From d65ab0a8035a0aaad8efeaea9bdaf211413aeecd Mon Sep 17 00:00:00 2001 From: Jaden Earl Date: Tue, 4 Aug 2026 10:58:02 -0600 Subject: [PATCH] Remove the source_archive subpackage Development moved to an internal repository. Co-Authored-By: Claude Fable 5 --- .env.template | 26 - .../test_source_archive/conftest.py | 59 -- .../test_source_archive/test_backends.py | 266 -------- .../test_source_archive/test_canonicalize.py | 82 --- .../test_source_archive/test_catalog.py | 139 ---- .../test_source_archive/test_content_store.py | 194 ------ .../test_source_archive/test_cost.py | 60 -- .../test_source_archive/test_coverage.py | 110 --- .../test_source_archive/test_layout.py | 123 ---- .../test_pipeline_and_manifest.py | 295 -------- .../test_quality_and_tiered.py | 61 -- .../test_source_archive/test_reindex.py | 87 --- .../test_source_archive/test_reports.py | 112 --- .../test_screenshot_encoding.py | 67 -- .../test_trace_extraction.py | 215 ------ .../test_url_extraction.py | 86 --- .../agents_and_tools/source_archive/README.md | 285 -------- .../source_archive/__init__.py | 56 -- .../source_archive/benchmark.py | 459 ------------- .../source_archive/canonicalize.py | 119 ---- .../source_archive/catalog.py | 571 ---------------- .../agents_and_tools/source_archive/cli.py | 353 ---------- .../agents_and_tools/source_archive/config.py | 88 --- .../source_archive/content_store.py | 323 --------- .../agents_and_tools/source_archive/cost.py | 137 ---- .../source_archive/coverage.py | 237 ------- .../source_archive/fetchers/__init__.py | 138 ---- .../source_archive/fetchers/base.py | 25 - .../fetchers/cloakbrowser_fetcher.py | 62 -- .../fetchers/firecrawl_fetcher.py | 122 ---- .../fetchers/hyperbrowser_fetcher.py | 149 ---- .../source_archive/fetchers/pdf_fetcher.py | 146 ---- .../fetchers/playwright_fetcher.py | 254 ------- .../source_archive/fetchers/tiered.py | 56 -- .../source_archive/ingest/__init__.py | 30 - .../source_archive/ingest/trace_extraction.py | 379 ----------- .../source_archive/ingest/url_extraction.py | 133 ---- .../agents_and_tools/source_archive/layout.py | 62 -- .../source_archive/manifest.py | 100 --- .../agents_and_tools/source_archive/models.py | 97 --- .../source_archive/pipeline.py | 317 --------- .../source_archive/quality.py | 56 -- .../source_archive/reindex.py | 278 -------- .../source_archive/reports.py | 93 --- .../source_archive/storage/__init__.py | 13 - .../source_archive/storage/blob_store.py | 25 - .../source_archive/storage/local_store.py | 34 - .../source_archive/storage/s3_store.py | 66 -- .../agents_and_tools/source_archive/viewer.py | 409 ----------- poetry.lock | 644 +----------------- pyproject.toml | 20 - 51 files changed, 3 insertions(+), 8315 deletions(-) delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/conftest.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_catalog.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_content_store.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_cost.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_coverage.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_layout.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_pipeline_and_manifest.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_quality_and_tiered.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reindex.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reports.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_screenshot_encoding.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_trace_extraction.py delete mode 100644 code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_url_extraction.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/README.md delete mode 100644 forecasting_tools/agents_and_tools/source_archive/__init__.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/benchmark.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/canonicalize.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/catalog.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/cli.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/config.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/content_store.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/cost.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/coverage.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/base.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/cloakbrowser_fetcher.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/firecrawl_fetcher.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/hyperbrowser_fetcher.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/pdf_fetcher.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/playwright_fetcher.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/fetchers/tiered.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/ingest/url_extraction.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/layout.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/manifest.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/models.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/pipeline.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/quality.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/reindex.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/reports.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/storage/__init__.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/storage/blob_store.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/storage/local_store.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/storage/s3_store.py delete mode 100644 forecasting_tools/agents_and_tools/source_archive/viewer.py diff --git a/.env.template b/.env.template index e8084ca7..349aff0f 100644 --- a/.env.template +++ b/.env.template @@ -18,33 +18,7 @@ METACULUS_API_BASE_URL=https://www.metaculus.com/api HUGGINGFACE_API_KEY= # As of Jun 10 2025, used for browser use agents. -# Also a fallback capture backend for the source archive (see below). HYPERBROWSER_API_KEY= -# --- Source archive (agents_and_tools/source_archive) ----------------------- -# Capture HTML + screenshot + markdown for every URL a bot cites. All optional; -# blank WEB_ARCHIVE_S3_BUCKET stores locally instead of S3. -WEB_ARCHIVE_S3_BUCKET= -WEB_ARCHIVE_S3_PREFIX=source-archive -WEB_ARCHIVE_AWS_PROFILE= -# Set to a local capture directory to run/view the archive with no S3 (the -# viewer reads from here when set). E.g. `capture --local ./archive`. -WEB_ARCHIVE_LOCAL_DIR= -WEB_ARCHIVE_TTL_DAYS=14 -# Managed fallback backends for the anti-bot / PDF tail behind self-hosted -# Playwright. FIRECRAWL also parses PDFs natively (OCR fallback for PdfFetcher). -FIRECRAWL_API_KEY= -# Firecrawl proxy mode for hardened anti-bot sites: basic (1 credit) | auto | -# stealth/enhanced (5 credits). Leave "basic" unless you need Cloudflare bypass. -WEB_ARCHIVE_FIRECRAWL_PROXY=basic -# Hyperbrowser session knobs (proxy turns a 1-credit scrape into 10 credits). -WEB_ARCHIVE_HYPERBROWSER_PROXY=true -WEB_ARCHIVE_HYPERBROWSER_STEALTH=true -WEB_ARCHIVE_HYPERBROWSER_CAPTCHA=true -# CloakBrowser (self-hosted anti-bot Playwright fork) module, if installed -# (`pip install cloakbrowser`). Exposes cloakbrowser.launch() -> Browser. -WEB_ARCHIVE_CLOAKBROWSER_IMPORT=cloakbrowser -WEB_ARCHIVE_PDF_MAX_PAGES=50 - # Disable if in Streamlit Cloud FILE_WRITING_ALLOWED=TRUE diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/conftest.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/conftest.py deleted file mode 100644 index ff07b829..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/conftest.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations - -import pytest - -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - - -class FakeFetcher: - """Returns canned CaptureResults by URL; raises FetchError for missing ones.""" - - name = "fake" - - def __init__(self) -> None: - self.responses: dict[str, CaptureResult] = {} - self.calls: list[str] = [] - - def add( - self, - url: str, - *, - html: str | None = None, - markdown: str | None = None, - status_code: int = 200, - screenshot: bytes | None = b"\x89PNG fake", - ) -> None: - body = ( - html - if html is not None - else "" + "content " * 80 + "" - ) - self.responses[url] = CaptureResult( - url=url, - final_url=url, - status_code=status_code, - html=body, - markdown=markdown if markdown is not None else "content " * 80, - screenshot=screenshot, - screenshot_content_type="image/png", - fetcher=self.name, - ) - - def fetch(self, url: str) -> CaptureResult: - self.calls.append(url) - if url not in self.responses: - raise FetchError(f"no canned response for {url}") - return self.responses[url] - - -@pytest.fixture -def make_fetcher(): - """Factory so a test can spin up one or several independent fake fetchers.""" - - def _factory(name: str = "fake") -> FakeFetcher: - f = FakeFetcher() - f.name = name - return f - - return _factory diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py deleted file mode 100644 index 53f0a6ac..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py +++ /dev/null @@ -1,266 +0,0 @@ -"""Unit tests for the backup capture backends and the bake-off pricing model. - -These mock the vendor SDKs so they run without API keys, network, browsers, or -the optional pymupdf/playwright/cloakbrowser packages installed. -""" - -from __future__ import annotations - -import base64 - -import pytest - -from forecasting_tools.agents_and_tools.source_archive import benchmark as B -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers import ( - build_default_fetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.fetchers.cloakbrowser_fetcher import ( - CloakBrowserFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.firecrawl_fetcher import ( - FirecrawlFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.hyperbrowser_fetcher import ( - HyperbrowserFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.pdf_fetcher import ( - PdfFetcher, - looks_like_pdf, -) -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - - -# --- Firecrawl proxy/stealth wiring ------------------------------------------ -def test_firecrawl_basic_sends_no_proxy_key(): - f = FirecrawlFetcher(ArchiveConfig(firecrawl_proxy="basic")) - assert "proxy" not in f._scrape_kwargs(["markdown"]) - - -@pytest.mark.parametrize("mode", ["auto", "stealth", "enhanced"]) -def test_firecrawl_stealth_sends_proxy_key(mode): - f = FirecrawlFetcher(ArchiveConfig(firecrawl_proxy=mode)) - assert f._scrape_kwargs(["markdown"])["proxy"] == mode - - -def test_firecrawl_fetch_pdf_markdown(): - class FakeClient: - def scrape(self, url, **kwargs): - assert kwargs["formats"] == ["markdown"] - return {"markdown": "# PDF body " + "x " * 200} - - f = FirecrawlFetcher(ArchiveConfig(firecrawl_api_key="k"), client=FakeClient()) - assert f.fetch_pdf_markdown("https://x/y.pdf").startswith("# PDF body") - - -# --- Hyperbrowser screenshot coercion + result mapping ----------------------- -def test_hyperbrowser_coerce_screenshot_data_uri(): - raw = b"\x89PNG fake" - uri = "data:image/png;base64," + base64.b64encode(raw).decode() - shot, ctype = HyperbrowserFetcher._coerce_screenshot(uri) - assert shot == raw and ctype == "image/png" - - -def test_hyperbrowser_coerce_screenshot_bare_base64(): - raw = b"\x89PNG fake" - shot, ctype = HyperbrowserFetcher._coerce_screenshot(base64.b64encode(raw).decode()) - assert shot == raw and ctype == "image/png" - - -def test_hyperbrowser_coerce_screenshot_none(): - assert HyperbrowserFetcher._coerce_screenshot(None) == (None, None) - - -def test_hyperbrowser_fetch_maps_result(monkeypatch): - class Data: - metadata = {"statusCode": 200, "title": "T", "sourceURL": "https://final"} - html = "ok" - markdown = "ok " * 100 - screenshot = None - - class Resp: - status = "completed" - error = None - data = Data() - - class FakeClient: - class scrape: - @staticmethod - def start_and_wait(params): - return Resp() - - f = HyperbrowserFetcher( - ArchiveConfig(hyperbrowser_api_key="k"), client=FakeClient() - ) - # Avoid constructing real SDK request models in the unit test. - monkeypatch.setattr(f, "_params", lambda url: None) - result = f.fetch("https://x") - assert result.fetcher == "hyperbrowser" - assert result.final_url == "https://final" - assert result.status_code == 200 - assert result.metadata["used_proxy"] is True - - -def test_hyperbrowser_failed_job_raises(monkeypatch): - class Resp: - status = "failed" - error = "blocked" - data = None - - class FakeClient: - class scrape: - @staticmethod - def start_and_wait(params): - return Resp() - - f = HyperbrowserFetcher( - ArchiveConfig(hyperbrowser_api_key="k"), client=FakeClient() - ) - monkeypatch.setattr(f, "_params", lambda url: None) - with pytest.raises(FetchError): - f.fetch("https://x") - - -def test_hyperbrowser_requires_key(): - with pytest.raises(FetchError): - HyperbrowserFetcher(ArchiveConfig(hyperbrowser_api_key=None)).fetch("https://x") - - -# --- PDF fetcher ------------------------------------------------------------- -def test_looks_like_pdf(): - assert looks_like_pdf("https://x/report.pdf") - assert looks_like_pdf("https://x/report.PDF?v=2") - assert not looks_like_pdf("https://x/report.html") - - -def test_pdf_rejects_non_pdf_bytes(): - f = PdfFetcher( - ArchiveConfig(), - downloader=lambda url, t: (b"not a pdf", url, 200), - ) - with pytest.raises(FetchError): - f.fetch("https://x/fake.pdf") - - -def test_pdf_falls_back_to_firecrawl_when_local_thin(monkeypatch): - class FakeFirecrawl: - def fetch_pdf_markdown(self, url): - return "# Scanned doc recovered by OCR " + "y " * 200 - - f = PdfFetcher( - ArchiveConfig(), - firecrawl=FakeFirecrawl(), - downloader=lambda url, t: (b"%PDF- minimal", url, 200), - ) - # Force the local parser to look thin regardless of whether pymupdf is present. - monkeypatch.setattr(f, "_parse_local", lambda data: (None, None, None, 3, "none")) - result = f.fetch("https://x/scan.pdf") - assert result.metadata["pdf_engine"] == "firecrawl" - assert "OCR" in result.markdown - - -def test_pdf_uses_local_when_text_is_rich(monkeypatch): - f = PdfFetcher( - ArchiveConfig(), - downloader=lambda url, t: (b"%PDF- minimal", url, 200), - ) - rich = "# Title\n" + "real body text " * 100 - monkeypatch.setattr( - f, "_parse_local", lambda data: (rich, b"png", "image/png", 5, "pymupdf4llm") - ) - result = f.fetch("https://x/clean.pdf") - assert result.metadata["pdf_engine"] == "pymupdf4llm" - assert result.metadata["pdf_pages"] == 5 - assert result.screenshot == b"png" - - -# --- CloakBrowser ------------------------------------------------------------ -def test_cloakbrowser_missing_package_gives_clear_error(monkeypatch): - # Force every import to fail so this passes whether or not cloakbrowser is - # actually installed in the test environment. - import forecasting_tools.agents_and_tools.source_archive.fetchers.cloakbrowser_fetcher as cb - - def _boom(name): - raise ImportError(name) - - monkeypatch.setattr(cb.importlib, "import_module", _boom) - f = CloakBrowserFetcher(ArchiveConfig()) - with pytest.raises(FetchError) as exc: - f._launch_browser() - assert "cloakbrowser" in str(exc.value).lower() - - -# --- Pricing model ----------------------------------------------------------- -def test_pricing_self_host_is_floor(): - r = CaptureResult(url="u", final_url="u") - assert B.estimate_cost("playwright", r, 1_000_000, B.Pricing()) == 0.00001 - assert B.estimate_cost("cloakbrowser", r, 1_000_000, B.Pricing()) == 0.00001 - - -def test_pricing_firecrawl_basic_vs_stealth(): - basic = CaptureResult(url="u", final_url="u", metadata={"firecrawl_proxy": "basic"}) - stealth = CaptureResult( - url="u", final_url="u", metadata={"firecrawl_proxy": "auto"} - ) - assert B.estimate_cost("firecrawl", basic, 0, B.Pricing()) == pytest.approx(0.00083) - assert B.estimate_cost( - "firecrawl-stealth", stealth, 0, B.Pricing() - ) == pytest.approx(0.00415) - - -def test_pricing_hyperbrowser_proxy_includes_bandwidth(): - r = CaptureResult(url="u", final_url="u", metadata={"used_proxy": True}) - # 10 credits ($0.01) + 1MB * $10/GB ($0.01) = $0.02 - assert B.estimate_cost("hyperbrowser", r, 1_000_000, B.Pricing()) == pytest.approx( - 0.02 - ) - - -def test_pricing_pdf_local_is_free_firecrawl_is_per_page(): - local = CaptureResult( - url="u", final_url="u", metadata={"pdf_engine": "pymupdf4llm"} - ) - ocr = CaptureResult( - url="u", final_url="u", metadata={"pdf_engine": "firecrawl", "pdf_pages": 10} - ) - assert B.estimate_cost("pdf", local, 0, B.Pricing()) == 0.0 - assert B.estimate_cost("pdf", ocr, 0, B.Pricing()) == pytest.approx(0.0083) - - -# --- Default tiered chain composition ---------------------------------------- -def _fake_browser(): - from unittest.mock import MagicMock - - return None, MagicMock() # (playwright_handle, browser) — browser.close() ok - - -def test_default_chain_cloakbrowser_is_primary(monkeypatch): - # CloakBrowser available -> it is the single self-hosted browser tier. - monkeypatch.setattr( - CloakBrowserFetcher, "_launch_browser", lambda self: _fake_browser() - ) - config = ArchiveConfig(hyperbrowser_api_key="h", firecrawl_api_key="f") - with build_default_fetcher(config) as fetcher: - names = [b.name for b in fetcher._tiered.backends] - # Note: exactly one browser tier (cloakbrowser), not vanilla + cloak. - assert names == ["cloakbrowser", "pdf", "firecrawl", "hyperbrowser"] - - -def test_default_chain_falls_back_to_playwright_and_skips_unkeyed(monkeypatch): - from forecasting_tools.agents_and_tools.source_archive.fetchers.playwright_fetcher import ( - PlaywrightFetcher, - ) - - # CloakBrowser not installed -> vanilla Playwright is the browser tier. - def raise_unavailable(self): - raise FetchError("cloakbrowser not installed") - - monkeypatch.setattr(CloakBrowserFetcher, "_launch_browser", raise_unavailable) - monkeypatch.setattr( - PlaywrightFetcher, "_launch_browser", lambda self: _fake_browser() - ) - config = ArchiveConfig(hyperbrowser_api_key=None, firecrawl_api_key=None) - with build_default_fetcher(config) as fetcher: - names = [b.name for b in fetcher._tiered.backends] - assert names == ["playwright", "pdf"] diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py deleted file mode 100644 index e45405c0..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import pytest - -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) -from forecasting_tools.agents_and_tools.source_archive.models import url_hash - -# (raw, expected canonical) — each pair documents one normalization rule. -CASES = [ - # fragment dropped - ("https://a.test/x#section", "https://a.test/x"), - # trailing slash dropped (non-root) - ("https://a.test/x/", "https://a.test/x"), - # root path collapses (with or without slash) to host only - ("https://a.test/", "https://a.test"), - ("https://a.test", "https://a.test"), - # scheme + host lowercased, path case preserved - ("HTTPS://A.TEST/Path", "https://a.test/Path"), - # default ports stripped, non-default kept - ("http://a.test:80/x", "http://a.test/x"), - ("https://a.test:443/x", "https://a.test/x"), - ("https://a.test:8443/x", "https://a.test:8443/x"), - # tracking params removed, meaningful params kept - ("https://a.test/x?utm_source=z&utm_medium=email", "https://a.test/x"), - ("https://a.test/x?id=7&fbclid=abc", "https://a.test/x?id=7"), - ("https://a.test/x?gclid=abc&igshid=q", "https://a.test/x"), - # remaining params sorted (order-independent) - ("https://a.test/x?b=2&a=1", "https://a.test/x?a=1&b=2"), - # bare "ref"/"source" are intentionally preserved - ("https://a.test/x?ref=home", "https://a.test/x?ref=home"), - # combination - ( - "HTTPS://A.TEST:443/Path/?b=2&utm_campaign=spring&a=1#frag", - "https://a.test/Path?a=1&b=2", - ), - # non-http(s) left alone - ("mailto:someone@a.test", "mailto:someone@a.test"), -] - - -@pytest.mark.parametrize("raw,expected", CASES) -def test_canonicalize_cases(raw: str, expected: str): - assert canonicalize_url(raw) == expected - - -@pytest.mark.parametrize("raw,_expected", CASES) -def test_canonicalize_is_idempotent(raw: str, _expected: str): - once = canonicalize_url(raw) - assert canonicalize_url(once) == once - - -def test_near_duplicates_share_a_url_hash(): - variants = [ - "https://a.test/article", - "https://a.test/article/", - "https://a.test/article#intro", - "https://a.test/article?utm_source=newsletter", - "HTTPS://A.test/article", - ] - hashes = {url_hash(v) for v in variants} - assert len(hashes) == 1 - - -def test_distinct_pages_keep_distinct_hashes(): - assert url_hash("https://a.test/x?id=1") != url_hash("https://a.test/x?id=2") - assert url_hash("https://a.test/x") != url_hash("https://a.test/y") - - -def test_empty_and_none_safe(): - assert canonicalize_url("") == "" - - -def test_lazy_port_valueerror_returns_raw() -> None: - # .port raises ValueError lazily when the port section is non-numeric — - # junk the URL regex can extract from CSS-ish text ("Port could not be - # cast to integer value as 'root{--novem-render-frac'"). canonicalize - # must not propagate it. - junk = "http://a.test:root{--novem-render-frac/x" - assert canonicalize_url(junk) == junk - assert canonicalize_url(canonicalize_url(junk)) == canonicalize_url(junk) diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_catalog.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_catalog.py deleted file mode 100644 index 3387d046..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_catalog.py +++ /dev/null @@ -1,139 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive import manifest as manifest_io -from forecasting_tools.agents_and_tools.source_archive.catalog import ( - build_catalog, - write_catalog, -) -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.models import ( - CaptureResult, - CitationRecord, -) -from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - -def _capture(url: str, html: str) -> CaptureResult: - return CaptureResult( - url=url, - final_url=url, - status_code=200, - html=html, - markdown="md " * 30, - screenshot=b"img", - screenshot_content_type="image/png", - fetcher="fake", - ) - - -def _seed(tmp_path): - store = LocalBlobStore(tmp_path) - config = ArchiveConfig(s3_prefix="t") - cstore = ContentStore(store, config) - cstore.store(_capture("https://a.test/p", "

a

")) - cstore.store(_capture("https://b.test/q", "

b

")) - # uncaptured.test/x is cited but never captured. - records = [ - CitationRecord( - url="https://a.test/p?utm_source=news", # canonicalizes to /p - run_id="r1", - bot="alpha", - question_id="100", - question_url="https://www.metaculus.com/questions/100/", - tool_name="web_search", - ), - CitationRecord( - url="https://b.test/q", - run_id="r1", - bot="beta", - question_id="100", - question_url="https://www.metaculus.com/questions/100/", - tool_name="page_fetch", - ), - CitationRecord( - url="https://uncaptured.test/x", - run_id="r1", - bot="alpha", - question_id="100", - ), - # A data/API call made only via run_code -> excluded from the catalog. - CitationRecord( - url="https://data.test/api?fmt=csv", - run_id="r1", - bot="beta", - question_id="100", - tool_name="run_code", - ), - ] - manifest_io.write_blob(store, "r1", records, config) - return store, config - - -def test_build_catalog_joins_and_canonicalizes(tmp_path): - store, config = _seed(tmp_path) - data = build_catalog(store, config) - - # The two a.test variants collapse to one source; the run_code API call is - # excluded (tool/API call, not a page). - urls = {s.canonical_url for s in data.sources} - assert urls == { - "https://a.test/p", - "https://b.test/q", - "https://uncaptured.test/x", - } - assert data.excluded.get("tool_call") == 1 - assert "https://data.test/api?fmt=csv" not in urls - captured = {s.canonical_url for s in data.sources if s.captured} - assert captured == {"https://a.test/p", "https://b.test/q"} - - by_q = data.by_question() - assert set(by_q) == {"100"} - assert len(by_q["100"]) == 3 - by_bot = data.by_bot() - assert set(by_bot) == {"alpha", "beta"} - - -def test_build_catalog_excludes_parse_raising_urls(tmp_path): - store, config = _seed(tmp_path) - # A bare "http://[" (junk extracted from a bot comment) makes urlsplit raise - # ValueError ("Invalid IPv6 URL"); it must count as malformed, not crash. - records = [ - CitationRecord(url="http://[", run_id="r2", bot="alpha", question_id="100"), - ] - manifest_io.write_blob(store, "r2", records, config) - - data = build_catalog(store, config) - assert data.excluded.get("malformed") == 1 - assert "http://[" not in {s.canonical_url for s in data.sources} - - summary = write_catalog(store, config) - assert summary.excluded.get("malformed") == 1 - - -def test_write_catalog_emits_views(tmp_path): - store, config = _seed(tmp_path) - summary = write_catalog(store, config) - - assert summary.sources == 3 - assert summary.captured == 2 - assert summary.questions == 1 - assert summary.excluded.get("tool_call") == 1 - - keys = set(store.list_keys("t/catalog/")) - assert "t/catalog/index.html" in keys - assert "t/catalog/READ_ME_FIRST.html" in keys - assert "t/catalog/by-question/100.html" in keys - assert "t/catalog/by-question/100.csv" in keys - assert "t/catalog/by-bot/alpha.html" in keys - assert "t/catalog/by-domain/a.test.html" in keys - - q_html = store.get("t/catalog/by-question/100.html").decode("utf-8") - assert "https://a.test/p" in q_html - assert "alpha" in q_html # bot tag present - # Local links are relative into the content store. - assert "../../content/" in q_html - - q_csv = store.get("t/catalog/by-question/100.csv").decode("utf-8") - assert "https://uncaptured.test/x" in q_csv - assert "no" in q_csv # uncaptured row marked diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_content_store.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_content_store.py deleted file mode 100644 index 27dfdf15..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_content_store.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.models import ( - CaptureResult, - url_hash, -) -from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - -def _store(tmp_path, **cfg) -> ContentStore: - return ContentStore(LocalBlobStore(tmp_path), ArchiveConfig(s3_prefix="t", **cfg)) - - -def _result(url: str, html: str, final_url: str | None = None) -> CaptureResult: - return CaptureResult( - url=url, - final_url=final_url if final_url is not None else url, - status_code=200, - html=html, - markdown="md " * 50, - screenshot=b"img", - screenshot_content_type="image/png", - fetcher="fake", - ) - - -def test_store_writes_blobs_and_index(tmp_path): - store = _store(tmp_path) - res = store.store(_result("https://a.test", "

one

")) - assert res.created is True - cap = res.capture - assert store.blobs.exists(cap.html_key) - assert store.blobs.exists(cap.markdown_key) - assert store.blobs.exists(cap.screenshot_key) - - -def test_lookup_within_ttl_is_cache_hit(tmp_path): - store = _store(tmp_path, ttl_days=14) - store.store(_result("https://a.test", "

one

")) - assert store.lookup("https://a.test") is not None - - -def test_lookup_after_ttl_expires_returns_none(tmp_path): - store = _store(tmp_path, ttl_days=14) - store.store(_result("https://a.test", "

one

")) - - uh = url_hash("https://a.test") - index = store._read_index(uh) - old = (datetime.now(timezone.utc) - timedelta(days=30)).isoformat() - for cap in index["captures"].values(): - cap["last_seen"] = old - store._write_index(uh, index) - - assert store.lookup("https://a.test") is None - - -def test_identical_content_is_deduped(tmp_path): - store = _store(tmp_path) - first = store.store(_result("https://a.test", "

same

")) - second = store.store(_result("https://a.test", "

same

")) - assert first.created is True - assert second.created is False - assert first.capture.content_hash == second.capture.content_hash - - -def test_changed_content_creates_new_capture(tmp_path): - store = _store(tmp_path) - first = store.store(_result("https://a.test", "

v1

")) - second = store.store(_result("https://a.test", "

v2 changed

")) - assert second.created is True - assert first.capture.content_hash != second.capture.content_hash - - -# --- Phase B: redirect aliasing ------------------------------------------- -def test_redirect_keys_capture_by_final_url(tmp_path): - store = _store(tmp_path) - res = store.store( - _result("https://bit.ly/x", "

dest

", final_url="https://dest.test/page") - ) - # Capture is stored under the FINAL url's hash, not the shortener's. - assert res.capture.url == "https://dest.test/page" - assert res.capture.url_hash == url_hash("https://dest.test/page") - # The canonical index records the cited shortener as an alias. - canonical = store._read_index(url_hash("https://dest.test/page")) - assert "https://bit.ly/x" in canonical["aliases"] - - -def test_lookup_via_shortener_and_final_both_hit(tmp_path): - store = _store(tmp_path) - store.store( - _result("https://bit.ly/x", "

dest

", final_url="https://dest.test/page") - ) - via_alias = store.lookup("https://bit.ly/x") - via_final = store.lookup("https://dest.test/page") - assert via_alias is not None and via_final is not None - assert via_alias.content_hash == via_final.content_hash - assert via_alias.url == "https://dest.test/page" - - -def test_two_shorteners_to_same_page_store_once(tmp_path): - store = _store(tmp_path) - first = store.store( - _result("https://bit.ly/x", "

same

", final_url="https://dest.test/page") - ) - second = store.store( - _result("https://t.co/y", "

same

", final_url="https://dest.test/page") - ) - assert first.created is True - assert second.created is False # identical content deduped, not re-stored - canonical = store._read_index(url_hash("https://dest.test/page")) - assert set(canonical["aliases"]) == {"https://bit.ly/x", "https://t.co/y"} - assert len(canonical["captures"]) == 1 - - -# --- Phase C: cross-URL content dedup ------------------------------------- -def test_identical_content_across_distinct_urls_reuses_blobs(tmp_path): - store = _store(tmp_path) - a = store.store(_result("https://a.test/x", "

same

")) - b = store.store(_result("https://b.test/y", "

same

")) - - # Both are real captures (each URL has its own index entry)... - assert a.created is True and b.created is True - # ...but B reuses A's blobs instead of writing its own. - assert a.capture.content_alias_of is None - assert b.capture.content_alias_of == url_hash("https://a.test/x") - assert b.capture.html_key == a.capture.html_key - - # No duplicate blob was written under B's url hash. - b_own_key = ( - f"t/content/{url_hash('https://b.test/y')}/{b.capture.content_hash}.html" - ) - assert not store.blobs.exists(b_own_key) - assert store.blobs.exists(a.capture.html_key) - - -def test_content_reverse_index_tracks_members(tmp_path): - store = _store(tmp_path) - store.store(_result("https://a.test/x", "

same

")) - store.store(_result("https://b.test/y", "

same

")) - - ch = store.store(_result("https://c.test/z", "

same

")).capture.content_hash - reverse = store._read_content_index(ch) - assert reverse["canonical_url_hash"] == url_hash("https://a.test/x") - member_hashes = {m["url_hash"] for m in reverse["members"]} - assert member_hashes == { - url_hash("https://a.test/x"), - url_hash("https://b.test/y"), - url_hash("https://c.test/z"), - } - - -def test_different_content_not_aliased(tmp_path): - store = _store(tmp_path) - a = store.store(_result("https://a.test/x", "

one

")) - b = store.store(_result("https://b.test/y", "

two different

")) - assert b.capture.content_alias_of is None - assert b.capture.html_key != a.capture.html_key - - -def test_incomplete_capture_is_not_a_cache_hit(tmp_path): - # A browser capture whose screenshot failed to encode (screenshot_key=None) - # is not "done" — the next run should retry it to fill the missing format. - store = _store(tmp_path, ttl_days=14) - store.store( - CaptureResult( - url="https://a.test", - final_url="https://a.test", - status_code=200, - html="

one

", - markdown="md " * 50, - screenshot=None, - fetcher="cloakbrowser", - ) - ) - assert store.lookup("https://a.test") is None - - -def test_pdf_capture_without_screenshot_is_complete(tmp_path): - # PDFs have no screenshot by nature, so markdown alone counts as complete. - store = _store(tmp_path, ttl_days=14) - store.store( - CaptureResult( - url="https://a.test/x.pdf", - final_url="https://a.test/x.pdf", - status_code=200, - markdown="md " * 50, - fetcher="pdf", - ) - ) - assert store.lookup("https://a.test/x.pdf") is not None diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_cost.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_cost.py deleted file mode 100644 index fe9dea55..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_cost.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.cost import ( - estimate_run_cost, - price_per_capture, -) -from forecasting_tools.agents_and_tools.source_archive.models import StoredCapture -from forecasting_tools.agents_and_tools.source_archive.pipeline import ( - CaptureOutcome, - PipelineSummary, -) - - -def _cap(url: str, fetcher: str) -> StoredCapture: - return StoredCapture(url=url, url_hash="h", content_hash="c", fetcher=fetcher) - - -def _stored(url: str, fetcher: str) -> CaptureOutcome: - return CaptureOutcome(url=url, status="stored", stored=_cap(url, fetcher)) - - -def test_free_backends_cost_nothing(): - cfg = ArchiveConfig() - for f in ("cloakbrowser", "playwright", "pdf", ""): - assert price_per_capture(f, cfg) == 0.0 - - -def test_paid_backends_priced_by_config(): - cfg = ArchiveConfig(hyperbrowser_use_proxy=True, firecrawl_proxy="basic") - assert price_per_capture("hyperbrowser", cfg) == 10 * 0.001 - assert price_per_capture("firecrawl", cfg) == 1 * 0.00083 - - cheap = ArchiveConfig(hyperbrowser_use_proxy=False, firecrawl_proxy="auto") - assert price_per_capture("hyperbrowser", cheap) == 1 * 0.001 - assert price_per_capture("firecrawl", cheap) == 5 * 0.00083 - - -def test_estimate_run_cost_breakdown(): - cfg = ArchiveConfig(hyperbrowser_use_proxy=True, firecrawl_proxy="basic") - summary = PipelineSummary( - outcomes=[ - _stored("u1", "cloakbrowser"), - _stored("u2", "cloakbrowser"), - _stored("u3", "hyperbrowser"), - _stored("u4", "firecrawl"), - CaptureOutcome( - url="u5", status="cache_hit", stored=_cap("u5", "cloakbrowser") - ), - CaptureOutcome(url="u6", status="error", reason="boom"), - ] - ) - rc = estimate_run_cost(summary, cfg, run_id="r1") - - assert rc.archived == 5 # 4 stored + 1 cache_hit; the error doesn't count - assert rc.paid_captures == 2 # hyperbrowser + firecrawl - assert rc.total_usd == round(0.01 + 0.00083, 4) # 0.0108 (4-dp rounding) - by = {b.backend: b for b in rc.by_backend} - assert by["cloakbrowser"].captures == 2 and by["cloakbrowser"].total_usd == 0.0 - assert by["hyperbrowser"].captures == 1 and by["hyperbrowser"].unit_usd == 0.01 diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_coverage.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_coverage.py deleted file mode 100644 index 155d3772..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_coverage.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive.catalog import Citation, Source -from forecasting_tools.agents_and_tools.source_archive.coverage import ( - coverage_from_sources, -) - - -def _src(url, domain, captured, cits): - return Source(canonical_url=url, domain=domain, captured=captured, citations=cits) - - -def _trace(bot, q, tool): - return Citation(bot=bot, question_id=q, tool_name=tool, origin="tool_result") - - -def _comment(bot, q): - return Citation(bot=bot, question_id=q, origin="metaculus_comment") - - -SOURCES = [ - _src( - "https://a.test/1", - "a.test", - True, - [_trace("template", "100", "scrape_webpage")], - ), - _src( - "https://b.test/2", - "b.test", - False, - [_trace("template", "100", "scrape_webpage")], - ), - _src("https://c.test/3", "c.test", True, [_comment("otherbot", "200")]), - # run_code-only -> excluded as a tool/API call - _src( - "https://data.test/x", - "data.test", - False, - [_trace("template", "100", "run_code")], - ), - # search-engine result page -> excluded as a non-source - _src( - "https://www.google.com/search?q=x", - "google.com", - False, - [_trace("template", "100", "scrape_webpage")], - ), - # malformed (extractor junk) -> excluded - _src( - "https://a.test/y%5B1%5D", - "a.test", - False, - [_trace("template", "100", "scrape_webpage")], - ), -] - - -def test_trace_report_excludes_non_sources_and_counts_pages(): - r = coverage_from_sources(SOURCES, "trace") - assert r.cited == 2 # a.test/1 + b.test/2 (data/search/malformed excluded) - assert r.captured == 1 - assert r.pct == 50.0 - assert r.excluded == {"tool_call": 1, "search": 1, "malformed": 1} - assert r.missing == 1 - assert r.missing_urls == ["https://b.test/2"] - - by_q = {row.label: (row.cited, row.captured) for row in r.by_question} - assert by_q == {"100": (2, 1)} - by_tool = {row.label: (row.cited, row.captured) for row in r.by_tool} - assert by_tool == {"scrape_webpage": (2, 1)} - missed = {row.label for row in r.missed_by_domain} - assert missed == {"b.test"} - - -def test_comment_report_is_separate(): - r = coverage_from_sources(SOURCES, "comments") - assert r.cited == 1 # only the metaculus_comment source - assert r.captured == 1 - assert r.pct == 100.0 - assert {row.label for row in r.by_bot} == {"otherbot"} - - -def test_modes_do_not_bleed(): - trace = coverage_from_sources(SOURCES, "trace") - comments = coverage_from_sources(SOURCES, "comments") - assert "https://c.test/3" not in trace.missing_urls # comment source not in trace - # the trace bot never appears in the comment report - assert "template" not in {row.label for row in comments.by_bot} - - -def test_csv_export_has_overall_row(): - csv_text = coverage_from_sources(SOURCES, "trace").to_csv() - assert "group,label,cited,captured,pct" in csv_text - assert "overall,trace,2,1,50.0" in csv_text - - -def test_outcomes_split_never_fetched_vs_failed(): - # b.test/2 is the only missing page source. With no outcome for it, it's a - # pure collection gap (never fetched). - r = coverage_from_sources(SOURCES, "trace", {"https://a.test/1": "stored"}) - assert r.has_outcomes is True - assert r.missing_never_fetched == 1 - assert r.missing_fetch_failed == 0 - - # If a run report shows b.test/2 was fetched and failed, it's a capture - # problem, not a collection gap. - r2 = coverage_from_sources(SOURCES, "trace", {"https://b.test/2": "error"}) - assert r2.missing_never_fetched == 0 - assert r2.missing_fetch_failed == 1 diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_layout.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_layout.py deleted file mode 100644 index 36147d38..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_layout.py +++ /dev/null @@ -1,123 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive import layout -from forecasting_tools.agents_and_tools.source_archive import manifest as manifest_io -from forecasting_tools.agents_and_tools.source_archive.catalog import _load_all_records -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.models import CitationRecord -from forecasting_tools.agents_and_tools.source_archive.pipeline import ( - CaptureOutcome, - PipelineSummary, -) -from forecasting_tools.agents_and_tools.source_archive.reports import ( - read_outcomes, - write_run_report, -) -from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - -# --- key helpers -------------------------------------------------------------- -def test_manifest_key_nests_by_group_daily_or_adhoc(): - assert ( - layout.manifest_key("r1", group="sprints/myrun") - == "manifests/sprints/myrun/r1.jsonl" - ) - assert ( - layout.manifest_key("daily-2026-07-01") - == "manifests/daily/2026-07/daily-2026-07-01.jsonl" - ) - assert layout.manifest_key("r1") == "manifests/adhoc/r1.jsonl" - - -def test_report_key_nests_and_keeps_suffix(): - assert ( - layout.report_key("daily-2026-07-01", ".json") - == "reports/daily/2026-07/daily-2026-07-01.json" - ) - assert layout.report_key("r1", "_cost.json") == "reports/adhoc/r1_cost.json" - assert ( - layout.report_key("r1", ".json", group="sprints/myrun") - == "reports/sprints/myrun/r1.json" - ) - - -def test_group_slashes_are_normalized(): - assert layout.manifest_key("r1", group="/sprints/myrun/") == ( - "manifests/sprints/myrun/r1.jsonl" - ) - - -def test_candidates_prefer_nested_then_legacy_flat(): - assert layout.manifest_key_candidates("r1") == [ - "manifests/adhoc/r1.jsonl", - "manifests/r1.jsonl", - ] - assert layout.report_key_candidates("r1", ".json") == [ - "reports/adhoc/r1.json", - "reports/r1.json", - ] - - -# --- readers ------------------------------------------------------------------ -def test_read_blob_falls_back_to_legacy_flat_key(tmp_path): - store = LocalBlobStore(tmp_path) - cfg = ArchiveConfig(s3_prefix="t") - legacy = manifest_io.dumps([CitationRecord(url="https://old.test", run_id="r1")]) - store.put("t/manifests/r1.jsonl", legacy.encode("utf-8")) - - assert manifest_io.read_blob(store, "r1", cfg)[0].url == "https://old.test" - - -def test_read_blob_prefers_nested_over_legacy(tmp_path): - store = LocalBlobStore(tmp_path) - cfg = ArchiveConfig(s3_prefix="t") - legacy = manifest_io.dumps([CitationRecord(url="https://old.test", run_id="r1")]) - store.put("t/manifests/r1.jsonl", legacy.encode("utf-8")) - manifest_io.write_blob( - store, "r1", [CitationRecord(url="https://new.test", run_id="r1")], cfg - ) - - assert manifest_io.read_blob(store, "r1", cfg)[0].url == "https://new.test" - - -def test_catalog_loads_nested_and_flat_manifests(tmp_path): - store = LocalBlobStore(tmp_path) - cfg = ArchiveConfig(s3_prefix="t") - store.put( - "t/manifests/old.jsonl", - manifest_io.dumps([CitationRecord(url="https://old.test")]).encode("utf-8"), - ) - manifest_io.write_blob( - store, "daily-2026-07-01", [CitationRecord(url="https://daily.test")], cfg - ) - manifest_io.write_blob( - store, - "sprint-run", - [CitationRecord(url="https://sprint.test")], - cfg, - group="sprints/myrun", - ) - - urls = {r.url for r in _load_all_records(store, "t")} - assert urls == {"https://old.test", "https://daily.test", "https://sprint.test"} - - -def test_read_outcomes_sees_nested_and_flat_reports(tmp_path): - store = LocalBlobStore(tmp_path) - cfg = ArchiveConfig(s3_prefix="t") - store.put( - "t/reports/old.json", - b'[{"url": "https://old.test", "status": "stored", "reason": ""}]', - ) - write_run_report( - store, - "daily-2026-07-01", - PipelineSummary( - outcomes=[CaptureOutcome(url="https://daily.test", status="error")] - ), - cfg, - ) - - out = read_outcomes(store, cfg) - assert out["https://old.test"] == "stored" - assert out["https://daily.test"] == "error" diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_pipeline_and_manifest.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_pipeline_and_manifest.py deleted file mode 100644 index ba895f4d..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_pipeline_and_manifest.py +++ /dev/null @@ -1,295 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive import manifest -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.models import CitationRecord -from forecasting_tools.agents_and_tools.source_archive.pipeline import ( - CapturePipeline, - capture_urls_concurrent, -) -from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - -def _pipeline(tmp_path, fetcher) -> CapturePipeline: - store = ContentStore( - LocalBlobStore(tmp_path), ArchiveConfig(s3_prefix="t", ttl_days=14) - ) - return CapturePipeline(fetcher, store) - - -def test_capture_urls_concurrent_captures_all(tmp_path, make_fetcher): - from contextlib import contextmanager - - config = ArchiveConfig(s3_prefix="t", concurrency=4) - store = ContentStore(LocalBlobStore(tmp_path), config) - urls = [f"https://s{i}.test/p" for i in range(12)] - - @contextmanager - def factory(_cfg): - f = make_fetcher() - for u in urls: - f.add(u) - yield f - - summary = capture_urls_concurrent(urls, store, config, factory) - - assert len(summary.outcomes) == 12 - assert summary.count("stored") == 12 - # every URL is resolvable afterwards (proves the shared store got all writes) - assert all(store.lookup(u) is not None for u in urls) - - -def test_concurrent_supervisor_recovers_a_stuck_worker(tmp_path, make_fetcher): - import threading - from contextlib import contextmanager - - config = ArchiveConfig(s3_prefix="t", concurrency=1) - store = ContentStore(LocalBlobStore(tmp_path), config) - urls = ["https://stuck.test/x"] - reaped = threading.Event() - builds = {"n": 0} - - class _Wedges: - name = "wedge" - - def fetch(self, url): - # Block until the supervisor's reaper "kills the browser", then surface - # the dead-browser error a killed Chromium would raise. - reaped.wait(5) - raise RuntimeError("Target page, context or browser has been closed") - - @contextmanager - def factory(_cfg): - builds["n"] += 1 - if builds["n"] == 1: - yield _Wedges() # first browser wedges - else: - fetcher = make_fetcher() - fetcher.add(urls[0]) - yield fetcher # rebuilt browser works - - # Inject a fake reaper so the test drives the supervisor without real Chromium. - summary = capture_urls_concurrent( - urls, store, config, factory, per_url_timeout=0.3, reaper=reaped.set - ) - - assert builds["n"] == 2 # stalled -> reaped -> death -> rebuild -> retry - assert summary.count("stored") == 1 # recovered and captured on a fresh browser - - -def test_concurrent_restarts_browser_after_death(tmp_path, make_fetcher): - from contextlib import contextmanager - - config = ArchiveConfig(s3_prefix="t", concurrency=1) - store = ContentStore(LocalBlobStore(tmp_path), config) - urls = ["https://a.test/x"] - builds = {"n": 0} - - class _DeadBrowser: - name = "dead" - - def fetch(self, url): - raise RuntimeError("Target page, context or browser has been closed") - - @contextmanager - def factory(_cfg): - builds["n"] += 1 - if builds["n"] == 1: - yield _DeadBrowser() # first browser is dead - else: - fetcher = make_fetcher() - fetcher.add(urls[0]) - yield fetcher # rebuilt browser works - - summary = capture_urls_concurrent(urls, store, config, factory) - - assert builds["n"] == 2 # detected death, rebuilt once - assert summary.count("stored") == 1 # retry on the fresh browser succeeded - - -def test_concurrent_rebuild_survives_poisoned_thread_loop_state(tmp_path, make_fetcher): - """Regression: a SIGKILLed sync-Playwright browser leaves its asyncio loop - registered as *running* on the worker thread (teardown is thread-affine, so - ``_close_quietly``'s helper thread can't clear it). The rebuild must reset - that thread-local state — otherwise ``sync_playwright().start()`` raises - "Sync API inside the asyncio loop" and future.result() kills the whole run. - """ - import asyncio - import threading - - config = ArchiveConfig(s3_prefix="t", concurrency=1) - store = ContentStore(LocalBlobStore(tmp_path), config) - urls = ["https://a.test/x", "https://b.test/y"] - builds = {"n": 0} - - class _SyncPlaywrightAlike: - """Mimics sync Playwright's thread behavior: __enter__ refuses if the - thread already reports a running loop, then registers its own; a killed - browser errors on fetch; teardown from a foreign thread fails the way a - greenlet does, leaving the loop state poisoned.""" - - name = "pwalike" - - def __init__(self, dead: bool, inner): - self._dead = dead - self._inner = inner - self._thread = None - - def __enter__(self): - try: - asyncio.get_running_loop() - except RuntimeError: - pass - else: # what playwright's sync context manager raises - raise RuntimeError( - "It looks like you are using Playwright Sync API inside " - "the asyncio loop." - ) - self._thread = threading.current_thread() - asyncio.events._set_running_loop(asyncio.new_event_loop()) - return self - - def fetch(self, url): - if self._dead: - raise RuntimeError("Target page, context or browser has been closed") - return self._inner.fetch(url) - - def __exit__(self, *exc): - if threading.current_thread() is not self._thread: - raise RuntimeError("cannot switch to a different thread") - asyncio.events._set_running_loop(None) - - def factory(_cfg): - builds["n"] += 1 - inner = make_fetcher() - for u in urls: - inner.add(u) - return _SyncPlaywrightAlike(dead=builds["n"] == 1, inner=inner) - - try: - summary = capture_urls_concurrent(urls, store, config, factory) - finally: - asyncio.events._set_running_loop(None) # never leak into other tests - - assert builds["n"] == 2 # death detected -> loop state reset -> rebuilt - assert summary.count("stored") == 2 # retried URL and the rest captured - - -def test_concurrent_failed_rebuild_does_not_kill_the_run(tmp_path, make_fetcher): - """If the rebuild itself fails (e.g. a transient launch error), the run must - keep going: the URL keeps its error outcome and the next URL retries the - rebuild.""" - from contextlib import contextmanager - - config = ArchiveConfig(s3_prefix="t", concurrency=1) - store = ContentStore(LocalBlobStore(tmp_path), config) - urls = ["https://a.test/x", "https://b.test/y"] - builds = {"n": 0} - - class _DeadBrowser: - name = "dead" - - def fetch(self, url): - raise RuntimeError("Target page, context or browser has been closed") - - @contextmanager - def factory(_cfg): - builds["n"] += 1 - if builds["n"] == 1: - yield _DeadBrowser() # first browser is dead - elif builds["n"] == 2: - raise RuntimeError("browser launch flaked") # rebuild attempt fails - else: - fetcher = make_fetcher() - for u in urls: - fetcher.add(u) - yield fetcher # second rebuild attempt works - - summary = capture_urls_concurrent(urls, store, config, factory) - - assert builds["n"] == 3 # dead -> failed rebuild -> successful rebuild - assert summary.count("error") == 1 # first URL kept its error outcome - assert summary.count("stored") == 1 # second URL captured after recovery - - -class _BoomFetcher: - """Raises an unexpected (non-FetchError) exception, like a bad screenshot.""" - - name = "boom" - - def fetch(self, url): - raise ValueError("kaboom") - - -def test_pipeline_isolates_unexpected_fetcher_errors(tmp_path): - # One pathological URL must not abort the whole run. - pipe = _pipeline(tmp_path, _BoomFetcher()) - summary = pipe.run(["https://a.test", "https://b.test"]) - assert summary.count("error") == 2 - assert len(summary.outcomes) == 2 - assert all(o.reason.startswith("unexpected:") for o in summary.outcomes) - - -def test_manifest_roundtrip_and_unique_urls(): - records = [ - CitationRecord(url="https://a.test", run_id="r1", bot="b", tool_name="search"), - CitationRecord(url="https://a.test", run_id="r1", bot="b", tool_name="fetch"), - CitationRecord(url="https://b.test", run_id="r1", bot="b"), - ] - back = manifest.loads(manifest.dumps(records)) - assert [r.url for r in back] == [r.url for r in records] - assert list(manifest.unique_urls(back)) == ["https://a.test", "https://b.test"] - - -def test_manifest_blob_roundtrip(tmp_path): - store = LocalBlobStore(tmp_path) - cfg = ArchiveConfig(s3_prefix="t") - records = [CitationRecord(url="https://a.test", run_id="r1")] - manifest.write_blob(store, "r1", records, cfg) - assert store.exists("t/manifests/adhoc/r1.jsonl") - assert manifest.read_blob(store, "r1", cfg)[0].url == "https://a.test" - - -def test_pipeline_stores_then_cache_hits(tmp_path, make_fetcher): - fetcher = make_fetcher() - fetcher.add("https://a.test") - pipeline = _pipeline(tmp_path, fetcher) - - first = pipeline.run(["https://a.test"]) - assert first.count("stored") == 1 - assert fetcher.calls == ["https://a.test"] - - second = pipeline.run(["https://a.test"]) - assert second.count("cache_hit") == 1 - assert fetcher.calls == ["https://a.test"] # not refetched - - -def test_pipeline_quality_failed_not_stored(tmp_path, make_fetcher): - fetcher = make_fetcher() - fetcher.add("https://bad.test", status_code=404) - pipeline = _pipeline(tmp_path, fetcher) - - summary = pipeline.run(["https://bad.test"]) - assert summary.count("quality_failed") == 1 - assert summary.captures == {} - - -def test_pipeline_error_when_no_backend_succeeds(tmp_path, make_fetcher): - fetcher = make_fetcher() # no canned responses -> FetchError - pipeline = _pipeline(tmp_path, fetcher) - summary = pipeline.run(["https://missing.test"]) - assert summary.count("error") == 1 - - -def test_pipeline_run_manifest_dedups_urls(tmp_path, make_fetcher): - fetcher = make_fetcher() - fetcher.add("https://a.test") - pipeline = _pipeline(tmp_path, fetcher) - records = [ - CitationRecord(url="https://a.test", tool_name="search"), - CitationRecord(url="https://a.test", tool_name="fetch"), - ] - summary = pipeline.run_manifest(records) - assert len(summary.outcomes) == 1 - assert fetcher.calls == ["https://a.test"] diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_quality_and_tiered.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_quality_and_tiered.py deleted file mode 100644 index d4f6b697..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_quality_and_tiered.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive.fetchers.tiered import ( - TieredFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult -from forecasting_tools.agents_and_tools.source_archive.quality import evaluate - - -def _cap(**kw) -> CaptureResult: - base = dict(url="u", final_url="u", status_code=200, html=None, markdown="x " * 200) - base.update(kw) - return CaptureResult(**base) - - -def test_quality_passes_real_page(): - assert evaluate(_cap()).passed - - -def test_quality_fails_404(): - assert not evaluate(_cap(status_code=404)).passed - - -def test_quality_fails_thin_content(): - assert not evaluate(_cap(markdown="short")).passed - - -def test_quality_fails_block_page(): - v = evaluate(_cap(markdown="Attention Required! | Cloudflare " * 20)) - assert not v.passed - assert "block_signature" in v.reason - - -def test_tiered_falls_back_to_secondary_on_quality_fail(make_fetcher): - primary = make_fetcher("primary") - primary.add("https://blocked.test", markdown="please enable javascript " * 20) - secondary = make_fetcher("secondary") - secondary.add("https://blocked.test") - - result = TieredFetcher(primary, secondary).fetch("https://blocked.test") - assert result.fetcher == "secondary" - assert result.metadata["quality_passed"] is True - - -def test_tiered_falls_back_on_fetch_error(make_fetcher): - primary = make_fetcher("primary") # no canned response -> FetchError - secondary = make_fetcher("secondary") - secondary.add("https://x.test") - - result = TieredFetcher(primary, secondary).fetch("https://x.test") - assert result.fetcher == "secondary" - - -def test_tiered_returns_failed_capture_when_all_fail(make_fetcher): - primary = make_fetcher("primary") - primary.add("https://x.test", status_code=404) - secondary = make_fetcher("secondary") - secondary.add("https://x.test", status_code=500) - - result = TieredFetcher(primary, secondary).fetch("https://x.test") - assert result.metadata["quality_passed"] is False diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reindex.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reindex.py deleted file mode 100644 index 82e5f5b7..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reindex.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import annotations - -import json - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.reindex import ( - analyze, - rebuild_content_index, -) -from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - -def _put_index(store, key: str, body: dict) -> None: - store.put(f"t/index/{key}.json", json.dumps(body).encode("utf-8")) - - -def _canonical(url: str, content_hash: str) -> dict: - return { - "url": url, - "url_hash": f"hash_of_{url}", - "latest_content_hash": content_hash, - "captures": { - content_hash: { - "url": url, - "url_hash": f"hash_of_{url}", - "content_hash": content_hash, - "html_key": f"t/content/hash_of_{url}/{content_hash}.html", - } - }, - } - - -def _seed(tmp_path) -> tuple[LocalBlobStore, ArchiveConfig]: - store = LocalBlobStore(tmp_path) - config = ArchiveConfig(s3_prefix="t") - # Legacy rows stored under raw hashing: two URLs that now canonicalize equal. - _put_index(store, "h1", _canonical("https://x.test/p?utm_source=news", "c1")) - _put_index(store, "h2", _canonical("https://x.test/p", "c2")) - # Two distinct URLs with byte-identical content (same latest hash). - _put_index(store, "h3", _canonical("https://a.test/1", "cX")) - _put_index(store, "h4", _canonical("https://b.test/2", "cX")) - # Same host+path, meaningful query differs -> Phase D candidate. - _put_index(store, "h5", _canonical("https://q.test/item?id=1", "n1")) - _put_index(store, "h6", _canonical("https://q.test/item?id=2", "n2")) - # An alias (redirect) index -> counted but not a capture. - _put_index(store, "h7", {"url": "https://bit.ly/z", "alias_of": "hash_of_x"}) - return store, config - - -def test_analyze_reports_all_three_lenses(tmp_path): - store, config = _seed(tmp_path) - report = analyze(store, config) - - assert report.total_url_indexes == 7 - assert report.alias_indexes == 1 - assert report.canonical_captures == 6 - - canon_keys = {c.key for c in report.canonicalization_clusters} - assert "https://x.test/p" in canon_keys - - content_urls = {tuple(c.urls) for c in report.content_clusters} - assert ("https://a.test/1", "https://b.test/2") in content_urls - - near_keys = {c.key for c in report.near_dup_clusters} - assert "https://q.test/item" in near_keys - - -def test_analyze_ignores_reverse_content_index(tmp_path): - store, config = _seed(tmp_path) - # A by-content reverse index must not be mistaken for a URL index. - store.put( - "t/index/by-content/cX.json", - json.dumps({"content_hash": "cX", "canonical_url_hash": "x"}).encode("utf-8"), - ) - report = analyze(store, config) - assert report.total_url_indexes == 7 # unchanged - - -def test_rebuild_content_index_is_dry_by_default(tmp_path): - store, config = _seed(tmp_path) - groups = rebuild_content_index(store, config, apply=False) - assert groups >= 1 - # Dry run wrote nothing under by-content/. - assert not list(store.list_keys("t/index/by-content/")) - - rebuild_content_index(store, config, apply=True) - assert list(store.list_keys("t/index/by-content/")) diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reports.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reports.py deleted file mode 100644 index 7a637a64..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_reports.py +++ /dev/null @@ -1,112 +0,0 @@ -from __future__ import annotations - -import json - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.cost import ( - RunCost, - write_cost_report, -) -from forecasting_tools.agents_and_tools.source_archive.models import ( - StoredCapture, - url_hash, -) -from forecasting_tools.agents_and_tools.source_archive.pipeline import ( - CaptureOutcome, - PipelineSummary, -) -from forecasting_tools.agents_and_tools.source_archive.reports import ( - read_outcomes, - write_run_report, -) -from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - -def _stored(url: str, fetcher: str) -> StoredCapture: - return StoredCapture( - url=url, url_hash=url_hash(url), content_hash="c1", fetcher=fetcher - ) - - -def test_run_report_roundtrip_canonicalizes(tmp_path): - store = LocalBlobStore(tmp_path) - config = ArchiveConfig(s3_prefix="t") - summary = PipelineSummary( - outcomes=[ - CaptureOutcome(url="https://a.test/p?utm_source=x", status="stored"), - CaptureOutcome(url="https://b.test/q", status="error", reason="cloudflare"), - ] - ) - write_run_report(store, "r1", summary, config) - - out = read_outcomes(store, config) - # keys are canonicalized (tracking param stripped) - assert out["https://a.test/p"] == "stored" - assert out["https://b.test/q"] == "error" - - -def test_run_report_records_backend_per_url(tmp_path): - store = LocalBlobStore(tmp_path) - config = ArchiveConfig(s3_prefix="t") - summary = PipelineSummary( - outcomes=[ - CaptureOutcome( - url="https://a.test/p", - status="stored", - stored=_stored("https://a.test/p", "firecrawl"), - ), - CaptureOutcome( - url="https://c.test/r", - status="cache_hit", - stored=_stored("https://c.test/r", "playwright"), - ), - CaptureOutcome(url="https://b.test/q", status="error", reason="cloudflare"), - ] - ) - key = write_run_report(store, "r1", summary, config) - - rows = {r["url"]: r for r in json.loads(store.get(key).decode("utf-8"))} - assert rows["https://a.test/p"]["backend"] == "firecrawl" - assert rows["https://c.test/r"]["backend"] == "playwright" - assert rows["https://b.test/q"]["backend"] == "" # nothing fetched - - -def test_captured_status_wins_over_failure(tmp_path): - store = LocalBlobStore(tmp_path) - config = ArchiveConfig(s3_prefix="t") - write_run_report( - store, - "early", - PipelineSummary( - outcomes=[CaptureOutcome(url="https://a.test", status="error")] - ), - config, - ) - write_run_report( - store, - "later", - PipelineSummary( - outcomes=[CaptureOutcome(url="https://a.test", status="stored")] - ), - config, - ) - assert read_outcomes(store, config)["https://a.test"] == "stored" - - -def test_read_outcomes_ignores_cost_reports(tmp_path): - """Cost reports live under reports/ too (``_cost.json``, a JSON - dict, not a list of rows) — read_outcomes must skip them, not crash.""" - store = LocalBlobStore(tmp_path) - config = ArchiveConfig(s3_prefix="t") - write_run_report( - store, - "r1", - PipelineSummary( - outcomes=[CaptureOutcome(url="https://a.test", status="stored")] - ), - config, - ) - write_cost_report(store, "r1", RunCost(run_id="r1", archived=1), config) - - out = read_outcomes(store, config) - assert out == {"https://a.test": "stored"} diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_screenshot_encoding.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_screenshot_encoding.py deleted file mode 100644 index d357982b..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_screenshot_encoding.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Tests for screenshot encoding + the height cap. - -Regression guard for a silent truncation bug: the height cap used to be applied -via Playwright's ``clip`` *without* ``full_page``, which is bounded by the -viewport and chopped tall pages down to a single screen. The cap is now enforced -by cropping the full-page render in Pillow — these tests pin that behavior. -""" - -from __future__ import annotations - -import io - -import pytest - -from forecasting_tools.agents_and_tools.source_archive.fetchers.playwright_fetcher import ( - _encode_screenshot, -) - -Image = pytest.importorskip("PIL.Image") - - -def _png(width: int, height: int) -> bytes: - out = io.BytesIO() - Image.new("RGB", (width, height), (255, 0, 0)).save(out, format="PNG") - return out.getvalue() - - -def test_tall_page_cropped_to_max_height(): - data, ct = _encode_screenshot(_png(1280, 12000), "webp", max_height=4000) - assert ct == "image/webp" - img = Image.open(io.BytesIO(data)) - assert img.size == (1280, 4000) # cropped to the cap, full width preserved - - -def test_short_page_not_cropped(): - data, _ = _encode_screenshot(_png(1280, 3000), "webp", max_height=20000) - assert Image.open(io.BytesIO(data)).size == (1280, 3000) # untouched - - -def test_webp_clamped_to_format_limit_even_without_cap(): - # WebP cannot exceed 16383px; an over-tall page must crop, not crash. - data, _ = _encode_screenshot(_png(1280, 25000), "webp", max_height=0) - assert Image.open(io.BytesIO(data)).size == (1280, 16383) - - -def test_webp_cap_above_format_limit_is_clamped(): - # A configured cap above WebP's limit still degrades safely to 16383. - data, _ = _encode_screenshot(_png(1280, 18000), "webp", max_height=16000) - assert Image.open(io.BytesIO(data)).height == 16000 - - -def test_png_keeps_full_height_uncapped(): - # PNG has no such limit, so max_height=0 preserves the whole render. - data, _ = _encode_screenshot(_png(1280, 20000), "png", max_height=0) - assert Image.open(io.BytesIO(data)).size == (1280, 20000) - - -def test_webp_is_real_webp(): - data, ct = _encode_screenshot(_png(800, 600), "webp") - assert ct == "image/webp" - assert data[:4] == b"RIFF" and data[8:12] == b"WEBP" - - -def test_jpeg_format(): - data, ct = _encode_screenshot(_png(800, 600), "jpeg") - assert ct == "image/jpeg" - assert Image.open(io.BytesIO(data)).format == "JPEG" diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_trace_extraction.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_trace_extraction.py deleted file mode 100644 index 2637eccd..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_trace_extraction.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from forecasting_tools.agents_and_tools.source_archive.ingest.trace_extraction import ( - extract_records_from_events, - extract_records_from_question_dir, - extract_records_from_trace_file, - harvest_run, - trace_label, -) - - -def test_trace_label_strips_prefix_and_suffix(): - assert trace_label("/x/traces_forecast_1_attempt_1.jsonl") == "forecast_1_attempt_1" - assert trace_label("traces_summarize.jsonl") == "summarize" - - -def test_tool_call_carries_query_and_tool_args(): - events = [ - { - "type": "tool_call", - "call_id": "c1", - "name": "search_online", - "args": {"query": "uk election polls", "max_results": 5}, - } - ] - records = extract_records_from_events(events, trace="forecast_1") - # No URL in the args -> nothing emitted from the tool_call itself. - assert records == [] - - -def test_tool_result_attributed_to_originating_call(): - events = [ - { - "type": "tool_call", - "call_id": "c1", - "name": "search_online", - "args": {"query": "uk election polls"}, - }, - { - "type": "tool_result", - "call_id": "c1", - "content": "Top hit: [poll](https://a.test/poll) and https://b.test/x", - "timestamp": "2026-05-12T12:00:00+00:00", - }, - ] - records = extract_records_from_events(events, trace="forecast_1", bot="template") - assert [r.url for r in records] == ["https://a.test/poll", "https://b.test/x"] - rec = records[0] - assert rec.origin == "tool_result" - assert rec.tool_name == "search_online" - assert rec.query == "uk election polls" - assert rec.tool_args == {"query": "uk election polls"} - assert rec.trace == "forecast_1" - assert rec.bot == "template" - assert rec.first_seen == "2026-05-12T12:00:00+00:00" - - -def test_query_from_list_args(): - events = [ - { - "type": "tool_call", - "call_id": "c1", - "name": "s", - "args": {"queries": ["a", "b"]}, - }, - {"type": "tool_result", "call_id": "c1", "content": "https://a.test/x"}, - ] - records = extract_records_from_events(events, trace="t") - assert records[0].query == "a b" - - -def test_url_directly_in_tool_call_args(): - events = [ - { - "type": "tool_call", - "call_id": "c1", - "name": "fetch_page", - "args": {"url": "https://a.test/page"}, - } - ] - records = extract_records_from_events(events, trace="t") - assert len(records) == 1 - assert records[0].url == "https://a.test/page" - assert records[0].origin == "tool_call" - assert records[0].tool_name == "fetch_page" - assert records[0].tool_args == {"url": "https://a.test/page"} - - -def test_initial_prompt_only_scanned_when_enabled(): - events = [ - {"type": "initial_prompt", "prompt": "background: https://a.test/bg"}, - ] - assert extract_records_from_events(events, trace="forecast_1") == [] - records = extract_records_from_events( - events, trace="summarize", include_initial_prompt=True - ) - assert [r.url for r in records] == ["https://a.test/bg"] - assert records[0].origin == "initial_prompt" - assert records[0].tool_name == "" - - -def test_non_dict_events_skipped(): - events = ["garbage", None, {"type": "tool_result", "content": "https://a.test/x"}] - records = extract_records_from_events(events, trace="t") - assert [r.url for r in records] == ["https://a.test/x"] - - -def _write_jsonl(path: Path, events: list[dict]) -> None: - path.write_text("\n".join(json.dumps(e) for e in events), encoding="utf-8") - - -def test_trace_file_uses_summarize_rule(tmp_path: Path): - f = tmp_path / "traces_summarize.jsonl" - _write_jsonl(f, [{"type": "initial_prompt", "prompt": "see https://a.test/r"}]) - records = extract_records_from_trace_file(str(f), run_id="run1", bot="template") - assert [r.url for r in records] == ["https://a.test/r"] - assert records[0].trace == "summarize" - assert records[0].run_id == "run1" - - -def test_trace_file_skips_blank_and_bad_lines(tmp_path: Path): - f = tmp_path / "traces_forecast_1.jsonl" - f.write_text( - '\n{"type": "tool_result", "content": "https://a.test/x"}\nnot json\n', - encoding="utf-8", - ) - records = extract_records_from_trace_file(str(f)) - assert [r.url for r in records] == ["https://a.test/x"] - - -def test_question_dir_reads_metadata_and_builds_url(tmp_path: Path): - qdir = tmp_path / "q_123" - qdir.mkdir() - (qdir / "question.json").write_text( - json.dumps({"question_id": "metac_123", "metaculus_id": 123}), - encoding="utf-8", - ) - _write_jsonl( - qdir / "traces_forecast_1.jsonl", - [{"type": "tool_result", "content": "https://a.test/x"}], - ) - records = extract_records_from_question_dir( - str(qdir), run_id="run1", bot="template" - ) - assert len(records) == 1 - rec = records[0] - assert rec.question_id == "metac_123" - assert rec.metaculus_id == "123" - assert rec.question_url == "https://www.metaculus.com/questions/123/" - - -def test_question_dir_without_metadata_still_emits(tmp_path: Path): - qdir = tmp_path / "q_x" - qdir.mkdir() - _write_jsonl( - qdir / "traces_forecast_1.jsonl", - [{"type": "tool_result", "content": "https://a.test/x"}], - ) - records = extract_records_from_question_dir(str(qdir)) - assert [r.url for r in records] == ["https://a.test/x"] - assert records[0].question_id is None - assert records[0].question_url is None - - -def test_harvest_run_walks_bot_and_question_dirs(tmp_path: Path): - run = tmp_path / "run_demo" - qdir = run / "bot_template" / "q_1" - qdir.mkdir(parents=True) - (qdir / "question.json").write_text( - json.dumps({"metaculus_id": 1}), encoding="utf-8" - ) - _write_jsonl( - qdir / "traces_forecast_1.jsonl", - [{"type": "tool_result", "content": "https://a.test/x"}], - ) - records = harvest_run(str(run)) - assert len(records) == 1 - rec = records[0] - assert rec.run_id == "run_demo" - assert rec.bot == "template" - assert rec.metaculus_id == "1" - - -def test_harvest_run_flat_layout_without_bot_dirs(tmp_path: Path): - # Flat layout: //traces_*.jsonl with no bot_* grouping. - run = tmp_path / "s3_backfill" - qdir = run / "2026-05-20_metac_43538" - qdir.mkdir(parents=True) - (qdir / "question.json").write_text( - json.dumps({"metaculus_id": 43538}), encoding="utf-8" - ) - _write_jsonl( - qdir / "traces_forecast_1.jsonl", - [{"type": "tool_result", "content": "https://a.test/x"}], - ) - records = harvest_run(str(run), bot="mybot") - assert len(records) == 1 - rec = records[0] - assert rec.bot == "mybot" # the flat-layout bot override - assert rec.metaculus_id == "43538" # still read from question.json - - -def test_harvest_run_flat_layout_defaults_bot_to_run_name(tmp_path: Path): - run = tmp_path / "myrun" - qdir = run / "q_only" - qdir.mkdir(parents=True) - _write_jsonl( - qdir / "traces_x.jsonl", - [{"type": "tool_result", "content": "https://a.test/y"}], - ) - records = harvest_run(str(run)) # no bot= -> defaults to run dir name - assert [r.bot for r in records] == ["myrun"] diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_url_extraction.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_url_extraction.py deleted file mode 100644 index 443578bb..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_url_extraction.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive.ingest.url_extraction import ( - dedupe_records, - extract_citation_records, - extract_urls, -) - - -def test_extracts_markdown_autolink_and_bare(): - text = ( - "See [the report](https://a.test/report) and " - "plus bare https://c.test/x for details." - ) - assert extract_urls(text) == [ - "https://a.test/report", - "https://b.test/page", - "https://c.test/x", - ] - - -def test_trims_trailing_punctuation(): - assert extract_urls("ends a sentence at https://a.test/path.") == [ - "https://a.test/path" - ] - assert extract_urls("(see https://a.test/path)") == ["https://a.test/path"] - - -def test_keeps_balanced_parens_in_url(): - text = "https://en.wikipedia.org/wiki/Forecasting_(disambiguation)" - assert extract_urls(text) == [ - "https://en.wikipedia.org/wiki/Forecasting_(disambiguation)" - ] - - -def test_dedupes_preserving_order(): - text = "https://a.test x https://b.test y https://a.test" - assert extract_urls(text) == ["https://a.test", "https://b.test"] - - -def test_strips_trailing_backslash_escape_residue(): - # Markdown often leaves a trailing backslash, e.g. "Zaporizhzhia\" - assert extract_urls("see https://a.test/search?q=Zaporizhzhia\\ ok") == [ - "https://a.test/search?q=Zaporizhzhia" - ] - - -def test_cuts_markdown_reference_tail_and_keeps_both_urls(): - # The bare scan can glue ")[10](other)" onto a real URL; the tail is cut so - # the first URL is clean, and the genuinely-separate second URL (itself a - # valid markdown link) is still extracted. Order follows pattern precedence - # (markdown links before bare URLs), so compare as a set. - text = "https://a.test/story?id=123)[10](https://b.test/other)" - assert set(extract_urls(text)) == { - "https://a.test/story?id=123", - "https://b.test/other", - } - - -def test_ignores_non_http_and_empty(): - assert extract_urls("ftp://a.test mailto:x@y.test nope") == [] - assert extract_urls(None) == [] - assert extract_urls("") == [] - - -def test_extract_citation_records_attaches_provenance(): - records = extract_citation_records( - "source: https://a.test/r", - run_id="r1", - bot="demo", - question_id="42", - origin="metaculus_comment", - ) - assert len(records) == 1 - rec = records[0] - assert rec.url == "https://a.test/r" - assert rec.run_id == "r1" - assert rec.bot == "demo" - assert rec.question_id == "42" - assert rec.origin == "metaculus_comment" - - -def test_dedupe_records_keeps_first(): - records = extract_citation_records("https://a.test https://a.test https://b.test") - deduped = dedupe_records(records) - assert [r.url for r in deduped] == ["https://a.test", "https://b.test"] diff --git a/forecasting_tools/agents_and_tools/source_archive/README.md b/forecasting_tools/agents_and_tools/source_archive/README.md deleted file mode 100644 index 5c368e32..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/README.md +++ /dev/null @@ -1,285 +0,0 @@ -# Source Archive - -Capture and preserve the web sources a forecasting bot relied on. For every -unique URL a bot cited, this captures **HTML + a full-page screenshot + -markdown** in a single page load and stores it with provenance, so a forecast -can be audited later even if the original page changes or disappears. - -## Why this exists - -A bot's forecast is only as trustworthy as the sources behind it, and those -sources rot: pages get edited, paywalled, or deleted. This package snapshots -each cited URL at the time it was used. - -It is built to be cheap at scale. Two ideas do the heavy lifting: - -- **Self-hosted rendering.** A single headless-Chromium page load produces all - three artifacts (HTML, screenshot, markdown), at a tiny fraction of the cost - of managed scraping APIs. A hosted fallback (Firecrawl) is used only for sites - that block headless browsers. -- **A content store with a TTL cache.** Bots re-forecast the same open question - every 20–30 minutes for weeks, citing the same pages each time. The store is - keyed by `url + content-hash`: a URL captured within the TTL is *not* refetched, - and identical content is *not* re-stored. So the first capture costs real money - and every re-run is nearly free. - -## Install - -The backends are optional, so they aren't pulled in by a default install: - -```bash -pip install "forecasting-tools[source-archive]" -playwright install chromium # one-time browser download -``` - -## Configure - -Configuration is read from the environment (see the project `.env.template`): - -| Variable | Purpose | Default | -| --- | --- | --- | -| `WEB_ARCHIVE_S3_BUCKET` | Destination S3 bucket. Blank → store locally. | — | -| `WEB_ARCHIVE_S3_PREFIX` | Key prefix within the bucket. | `source-archive` | -| `WEB_ARCHIVE_AWS_PROFILE` | Named AWS profile (e.g. an SSO profile). | default chain | -| `WEB_ARCHIVE_TTL_DAYS` | Days before a cached capture is refetched. | `14` | -| `FIRECRAWL_API_KEY` | Enables the Firecrawl fallback. | — (fallback off) | -| `WEB_ARCHIVE_FIRECRAWL_PROXY` | Firecrawl proxy mode for hardened sites: `basic` (1 credit) / `auto` / `stealth` (5 credits). | `basic` | -| `HYPERBROWSER_API_KEY` | Enables the Hyperbrowser managed fallback. | — (off) | -| `WEB_ARCHIVE_CLOAKBROWSER_IMPORT` | Module exposing CloakBrowser's `launch()`. | `cloakbrowser` | -| `WEB_ARCHIVE_PDF_MAX_PAGES` | Cap on PDF pages parsed per document. | `50` | - -AWS credentials use the standard AWS resolution chain — environment variables, a -shared config file, or an SSO profile. Nothing secret is committed or baked into -the code. - -## Use it from Python - -```python -from forecasting_tools.agents_and_tools.source_archive import ( - ArchiveConfig, CapturePipeline, ContentStore, build_default_fetcher, -) -from forecasting_tools.agents_and_tools.source_archive.storage import ( - LocalBlobStore, S3BlobStore, -) - -config = ArchiveConfig.from_env() - -# Store locally while experimenting... -store = ContentStore(LocalBlobStore("./archive"), config) -# ...or to S3 in production: -# store = ContentStore(S3BlobStore(config.s3_bucket, config=config), config) - -with build_default_fetcher(config) as fetcher: - summary = CapturePipeline(fetcher, store).run([ - "https://example.com", - "https://www.federalregister.gov/", - ]) - -print(summary) -# PipelineSummary(total=2, cache_hit=0, stored=2, deduped=0, quality_failed=0, error=0) -``` - -## Use it from the command line - -```bash -# Inspect the resolved configuration (secrets are masked) -source-archive check - -# Capture every URL in a manifest, storing locally (no AWS needed) -source-archive capture run.jsonl --local ./archive - -# Capture and upload to S3 (uses WEB_ARCHIVE_S3_BUCKET), plus the manifest itself -source-archive capture run.jsonl --upload-manifest --run-id 2026-06-01_demo - -# Skip the Hyperbrowser fallback this run; failures are written to a retry -# manifest so you can come back to just those sites later (e.g. with it on). -source-archive capture run.jsonl --no-hyperbrowser --run-id demo -source-archive capture demo_needs_retry.jsonl --run-id demo # later, hyperbrowser on - -# Build a manifest by harvesting the URLs bots cited on a Metaculus tournament -source-archive harvest 32506 --out run.jsonl -``` - -Because a failed fetch leaves no cache entry while a success does, re-running the -same manifest only re-attempts the failures — the retry manifest just makes that -explicit and fast (it skips the already-captured majority). - -`source-archive` is installed by the extra; the equivalent module form is -`python -m forecasting_tools.agents_and_tools.source_archive.cli`. - -## Backup backends & the bake-off - -A self-hosted browser is the primary backend and gets ~70% of URLs for ~free, -but two kinds of URL fall through it: **anti-bot/Cloudflare** pages (it detects -the block but can't get past it) and **PDFs** (Chromium downloads them instead of -rendering, so nothing is captured). The package ships these backups, ordered by -marginal cost so the cheap tiers absorb most of the tail: - -| Backend | Cost (2026) | Closes | Notes | -| --- | --- | --- | --- | -| `CloakBrowserFetcher` | ~$0/page (self-host) | Cloudflare | **The primary browser tier when installed** (`pip install cloakbrowser`): patched Chromium that beat vanilla Playwright on Cloudflare in 2026 benchmarks. Only one browser runs — cloak *replaces* vanilla Playwright (two `sync_playwright` instances conflict in one process), falling back to vanilla when cloak isn't installed. | -| `PdfFetcher` | $0 local; ~$0.0008/pg OCR | PDFs | PyMuPDF4LLM locally, falls back to Firecrawl OCR on scanned PDFs. | -| `FirecrawlFetcher` | $0.0008 basic / $0.0042 stealth | Cloudflare + PDFs | Native PDF parser; `WEB_ARCHIVE_FIRECRAWL_PROXY=stealth` for hardened sites. | -| `HyperbrowserFetcher` | $0.001 basic / $0.01 proxy | Cloudflare | Consolidates spend onto a vendor already used elsewhere. No PDF support. | - -Selenium was evaluated and **rejected**: it drives the same Chromium as -Playwright, so it bypasses nothing Playwright can't, and its stealth ecosystem -(`undetected-chromedriver`) is now legacy. CloakBrowser/Patchright/nodriver are -the credible self-hosted upgrades. - -To decide which backup(s) to wire in, run the bake-off — it runs each selected -backend independently over the same URLs (not tiered) and reports reliability, -latency, and estimated cost per backend, broken down by category: - -```bash -python -m forecasting_tools.agents_and_tools.source_archive.benchmark \ - --manifest forecasting_tools/agents_and_tools/source_archive/benchmarks/sample_urls.jsonl \ - --backends playwright,cloakbrowser,firecrawl,firecrawl-stealth,hyperbrowser,pdf \ - --out bench.csv -``` - -Backends whose API key or dependency is missing are skipped cleanly. Cost -figures are model estimates (see `PRICING` in `benchmark.py`); tune the credit -rates with `--firecrawl-credit-usd` / `--hyperbrowser-credit-usd` to match your -plan. Swap the sample manifest for a JSONL of your own cited URLs (one -`{"url", "category"}` per line; categories `normal`/`cloudflare`/`pdf`) for a -representative run. - -## Browse what you captured - -A Streamlit viewer reads the manifests + index back out of the store and shows -each captured URL's **screenshot, markdown, and HTML** side by side, filterable -by bot and question: - -```bash -AWS_PROFILE=default WEB_ARCHIVE_S3_BUCKET=my-web-archive \ - streamlit run forecasting_tools/agents_and_tools/source_archive/viewer.py -``` - -It uses the same `ArchiveConfig.from_env()` settings as capture, so it points at -whatever bucket/prefix you captured to (no extra configuration). - -To browse a **local** capture (no S3/AWS), set `WEB_ARCHIVE_LOCAL_DIR` to the -directory you captured into with `--local`: - -```bash -WEB_ARCHIVE_LOCAL_DIR=./archive \ - streamlit run forecasting_tools/agents_and_tools/source_archive/viewer.py -``` - -## The catalog: a browsable, coworker-legible view - -The viewer is interactive (good for us); the **catalog** is a set of static -HTML/CSV pages written into the bucket so a non-technical coworker can browse the -sources without any tooling. It is **question-primary** — the encyclopedia of -every web source used for a question — plus `by-bot/` and `by-domain/` -cross-views, built by joining the manifests with the index: - -```bash -# write catalog/ into the bucket (uses WEB_ARCHIVE_S3_BUCKET) -source-archive catalog -# or against a local capture dir -source-archive catalog --local ./archive -``` - -Start at `catalog/index.html` (or `catalog/READ_ME_FIRST.html` for the plain -explainer). Each source shows its screenshot, who used it (bot + tool), and -whether it was captured; each question also has a CSV. Data/API calls (a bot's -`run_code` pulling a CSV, etc.) are **excluded** from the catalog — it lists web -pages a bot read, not data endpoints — though they remain in the raw manifests. - -## Coverage: what fraction did we archive? - -The catalog shows what we *have*; the **coverage report** shows what we're -*missing*. It's two separate reports, by ingestion path — different denominators, -different ground truth: - -```bash -source-archive coverage # both reports -source-archive coverage --mode trace # just the complex/template bot -source-archive coverage --csv ./cov # also write cov_.csv (+ _missing.txt) -``` - -- **trace** — the complex/template bot's instrumented runs (metac-ai-sdk). Traces - hold *every* URL the bot touched, so this is a true archival success-rate. -- **comments** — every bot (Metaculus's own + outsiders) harvested from public - comments. Comments are truncated, so this denominator under-counts — coverage - here means "of the links visible in comments, how many we archived." - -The report is oriented to one question: **are there sources bots are using that -we are not yet archiving?** It leads with that gap, then breaks it down by -question, bot, tool, and the biggest-gap sites, plus the list of sources to -collect. Non-source URLs — search-engine results, `run_code`-style tool/API -calls, and malformed extractor junk — are excluded (same as the catalog). - -If capture runs have persisted their outcomes (`reports/.json`, written -automatically by `capture`), the gap is split into **never fetched** (the real -collection gap) vs **fetched but failed** (a capture problem). - -## The manifest: what to feed it - -A run produces a **citation manifest** — a JSONL file with one record per cited -URL. Only `url` is required; the rest is provenance you fill in where you have it: - -```json -{"url": "https://example.com/report", "run_id": "2026-06-01_demo", "bot": "my-bot", "question_id": "1234", "question_url": "https://www.metaculus.com/questions/1234/", "tool_name": "web_search", "origin": "research"} -``` - -The pipeline dedupes URLs within the manifest before fetching. - -## Where the manifest comes from - -You write a manifest yourself, or generate one from a bot's recorded reasoning. - -**From text.** `extract_urls(text)` / `extract_citation_records(...)` in -`ingest.url_extraction` pull URLs out of any markdown/text (markdown links, -autolinks, and bare URLs) — point them at whatever record of a bot's reasoning -you have. - -**From instrumented traces.** For bots you control, a trace is the fullest -source. `source-archive ingest-traces ` walks a traced run and emits a -manifest of every URL the bot touched, with provenance (trace, tool, search -query). - -## How it's organized - -| Module | Responsibility | -| --- | --- | -| `config.py` | Environment-driven `ArchiveConfig` | -| `models.py` | `CaptureResult`, `StoredCapture`, `CitationRecord` | -| `ingest/` | Build a manifest: URL extraction from text + traced bot runs | -| `fetchers/` | Playwright (primary) + CloakBrowser / PDF / Firecrawl / Hyperbrowser backups, tiered orchestrator | -| `benchmark.py` | Backend bake-off: reliability + cost per backend over a manifest | -| `quality.py` | Reject 404s, block pages, and thin content before archiving | -| `storage/` | `BlobStore` interface with S3 and local backends | -| `content_store.py` | `url + content-hash` store with the TTL cache and dedup | -| `manifest.py` | Read/write citation manifests | -| `pipeline.py` | `lookup → fetch → quality gate → store` | -| `cli.py` | `source-archive` command | - -## Roadmap - -Planned and shipped improvements — smarter dedup (URL canonicalization + -redirect/content aliasing), the coworker-legible catalog, and coverage reports — -are written up in [ROADMAP.md](ROADMAP.md). - -## What lands in storage - -``` -/index/.json per-URL capture history (+ aliases) -/index/by-content/.json reverse index for content dedup -/content//.html -/content//.webp (screenshot) -/content//.md -/manifests//.jsonl the run's citation manifest -/reports//.json per-URL capture outcomes (for coverage) -/reports//_cost.json the run's estimated cost breakdown -/catalog/index.html browsable catalog (by question/bot/site) -/catalog/by-question/.{html,csv} -``` - -`` keeps manifests/reports from piling into one flat folder: it is -`daily/` for `daily-YYYY-MM-DD` run ids, `adhoc` otherwise, or -whatever you pin with `--group` (e.g. `--group sprints/myrun`). Readers list by -prefix (and fall back to the old flat keys), so archives written before this -layout keep working unchanged. diff --git a/forecasting_tools/agents_and_tools/source_archive/__init__.py b/forecasting_tools/agents_and_tools/source_archive/__init__.py deleted file mode 100644 index 5ede914d..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Source Archive — capture and store the web sources a forecasting bot cited. - -For every unique URL a bot used, this captures **HTML + screenshot + markdown** -in a single page load and stores it with provenance, deduplicated by -``url + content-hash`` so re-runs of the same question are nearly free. - -Quick start (see ``README.md`` in this package for the full guide):: - - from forecasting_tools.agents_and_tools.source_archive import ( - ArchiveConfig, CapturePipeline, ContentStore, build_default_fetcher, - ) - from forecasting_tools.agents_and_tools.source_archive.storage import LocalBlobStore - - config = ArchiveConfig.from_env() - store = ContentStore(LocalBlobStore("./archive"), config) - with build_default_fetcher(config) as fetcher: - summary = CapturePipeline(fetcher, store).run(["https://example.com"]) - print(summary) - -The heavy backends (Playwright, boto3, Firecrawl, trafilatura) are optional; -install them with ``pip install forecasting-tools[source-archive]``. -""" - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ( - ContentStore, - StoreResult, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers import ( - build_default_fetcher, -) -from forecasting_tools.agents_and_tools.source_archive.ingest import extract_urls -from forecasting_tools.agents_and_tools.source_archive.models import ( - CaptureResult, - CitationRecord, - StoredCapture, -) -from forecasting_tools.agents_and_tools.source_archive.pipeline import ( - CaptureOutcome, - CapturePipeline, - PipelineSummary, -) - -__all__ = [ - "ArchiveConfig", - "CaptureOutcome", - "CaptureResult", - "CapturePipeline", - "CitationRecord", - "ContentStore", - "PipelineSummary", - "StoreResult", - "StoredCapture", - "build_default_fetcher", - "extract_urls", -] diff --git a/forecasting_tools/agents_and_tools/source_archive/benchmark.py b/forecasting_tools/agents_and_tools/source_archive/benchmark.py deleted file mode 100644 index 76d79083..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/benchmark.py +++ /dev/null @@ -1,459 +0,0 @@ -"""Backend bake-off: run each capture backend independently over the same URLs. - -This is the harness for deciding *which* backup to put behind Playwright. Unlike -the production :class:`TieredFetcher` (which stops at the first backend that -passes the quality gate), the benchmark runs **every** selected backend over -**every** URL, so you get an apples-to-apples table of reliability, latency, and -estimated cost per backend — broken down by URL category (normal / cloudflare / -pdf). - -Run it:: - - python -m forecasting_tools.agents_and_tools.source_archive.benchmark \\ - --manifest sample_urls.jsonl \\ - --backends playwright,cloakbrowser,firecrawl,firecrawl-stealth,hyperbrowser,pdf \\ - --out bench.csv - -A backend whose dependency or API key is missing is skipped with a note rather -than failing the whole run, so you can benchmark whatever you have configured. - -Cost figures are ESTIMATES from a documented pricing model (see ``PRICING``, -sourced 2026-06); they are not billed amounts. Override the credit rates via -CLI flags to match your plan. -""" - -from __future__ import annotations - -import argparse -import csv -import io -import json -import logging -import statistics -import sys -import time -from contextlib import nullcontext -from dataclasses import dataclass, field -from pathlib import Path -from typing import Callable - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import ( - Fetcher, - FetchError, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.cloakbrowser_fetcher import ( - CloakBrowserFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.firecrawl_fetcher import ( - FirecrawlFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.hyperbrowser_fetcher import ( - HyperbrowserFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.pdf_fetcher import ( - PdfFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.playwright_fetcher import ( - PlaywrightFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult -from forecasting_tools.agents_and_tools.source_archive.quality import evaluate - -logger = logging.getLogger(__name__) - -GB = 1_000_000_000 - -# --- Pricing model ----------------------------------------------------------- -# $/unit as of 2026-06, from each vendor's public pricing + this repo's prior -# cost experiment. These are the knobs to adjust for your plan. -# -# - Self-hosted compute (Playwright / CloakBrowser): ~$0.00001/page rendered -# (measured in bot-sources probe). Marginal service fee is effectively $0. -# - Firecrawl: 1 credit basic, 5 credits stealth/"enhanced" proxy. Standard -# plan ≈ $0.00083/credit. -# - Hyperbrowser: 1 credit ($0.001) basic, 10 credits ($0.01) with proxy, -# plus $10/GB proxy bandwidth. 1 credit = $0.001. -# - PDF: PyMuPDF4LLM local = $0; Firecrawl OCR fallback = ~1 credit/PDF page. - - -@dataclass -class Pricing: - self_host_per_page: float = 0.00001 - firecrawl_credit_usd: float = 0.00083 - firecrawl_basic_credits: int = 1 - firecrawl_stealth_credits: int = 5 - hyperbrowser_credit_usd: float = 0.001 - hyperbrowser_basic_credits: int = 1 - hyperbrowser_proxy_credits: int = 10 - hyperbrowser_bandwidth_usd_per_gb: float = 10.0 - - -def estimate_cost( - backend: str, result: CaptureResult, response_bytes: int, pricing: Pricing -) -> float: - """Estimated $ for one successful capture by ``backend``.""" - meta = result.metadata or {} - if backend in ("playwright", "cloakbrowser"): - return pricing.self_host_per_page - if backend.startswith("firecrawl"): - proxy = str(meta.get("firecrawl_proxy", "basic")).lower() - credits = ( - pricing.firecrawl_basic_credits - if proxy in ("", "basic") - else pricing.firecrawl_stealth_credits - ) - return credits * pricing.firecrawl_credit_usd - if backend == "hyperbrowser": - credits = ( - pricing.hyperbrowser_proxy_credits - if meta.get("used_proxy") - else pricing.hyperbrowser_basic_credits - ) - bandwidth = (response_bytes / GB) * pricing.hyperbrowser_bandwidth_usd_per_gb - return credits * pricing.hyperbrowser_credit_usd + bandwidth - if backend == "pdf": - if meta.get("pdf_engine") == "firecrawl": - pages = int(meta.get("pdf_pages") or 1) - return pages * pricing.firecrawl_credit_usd - return 0.0 # local PyMuPDF4LLM - return 0.0 - - -# --- Backend registry -------------------------------------------------------- -# Factories so a missing dependency / API key only skips that backend. The -# ``context`` flag marks browser backends that must be entered as a context -# manager (the browser launches once and is reused across URLs). - - -@dataclass -class BackendSpec: - name: str - factory: Callable[[ArchiveConfig], Fetcher] - context: bool = False - # Optional pre-flight: return a reason string if the backend can't run - # (missing key/dep) so the bake-off reports a clean SKIP instead of N/N - # fetch_errors. ``None`` means "looks runnable". - precheck: Callable[[ArchiveConfig], str | None] | None = None - - -def _need_firecrawl_key(config: ArchiveConfig) -> str | None: - if not config.firecrawl_api_key: - return "FIRECRAWL_API_KEY not set" - return None - - -def _need_hyperbrowser_key(config: ArchiveConfig) -> str | None: - if not config.hyperbrowser_api_key: - return "HYPERBROWSER_API_KEY not set" - return None - - -def _firecrawl_stealth(config: ArchiveConfig) -> FirecrawlFetcher: - # Force the proxy/stealth path so this row measures the Cloudflare-grade - # (5-credit) cost, even if the operator left the default at "basic". - proxy = config.firecrawl_proxy - if proxy in ("", "basic"): - proxy = "auto" - f = FirecrawlFetcher(config.model_copy(update={"firecrawl_proxy": proxy})) - f.name = "firecrawl-stealth" - return f - - -BACKENDS: dict[str, BackendSpec] = { - "playwright": BackendSpec("playwright", PlaywrightFetcher, context=True), - "cloakbrowser": BackendSpec("cloakbrowser", CloakBrowserFetcher, context=True), - "firecrawl": BackendSpec( - "firecrawl", FirecrawlFetcher, precheck=_need_firecrawl_key - ), - "firecrawl-stealth": BackendSpec( - "firecrawl-stealth", _firecrawl_stealth, precheck=_need_firecrawl_key - ), - "hyperbrowser": BackendSpec( - "hyperbrowser", HyperbrowserFetcher, precheck=_need_hyperbrowser_key - ), - "pdf": BackendSpec("pdf", PdfFetcher), -} - - -# --- Sample manifest --------------------------------------------------------- -# A curated starter set spanning the three categories the backup must handle. -# Replace/extend with your own real cited URLs for a representative run. -SAMPLE_MANIFEST: list[dict] = [ - {"url": "https://example.com", "category": "normal"}, - {"url": "https://en.wikipedia.org/wiki/Forecasting", "category": "normal"}, - {"url": "https://www.federalregister.gov/", "category": "normal"}, - # Sites commonly fronted by Cloudflare / anti-bot: - {"url": "https://www.g2.com/", "category": "cloudflare"}, - {"url": "https://www.indeed.com/", "category": "cloudflare"}, - {"url": "https://www.zillow.com/", "category": "cloudflare"}, - # PDFs (the gap Playwright can't render): - {"url": "https://arxiv.org/pdf/1706.03762", "category": "pdf"}, - {"url": "https://bitcoin.org/bitcoin.pdf", "category": "pdf"}, -] - - -@dataclass -class Row: - backend: str - url: str - category: str - passed: bool - reason: str - seconds: float - html_bytes: int - md_bytes: int - screenshot_bytes: int - cost_usd: float - error: str = "" - - -@dataclass -class BackendRun: - name: str - rows: list[Row] = field(default_factory=list) - skipped: str = "" - - -def _sizes(result: CaptureResult) -> tuple[int, int, int]: - html = len(result.html.encode()) if result.html else 0 - md = len(result.markdown.encode()) if result.markdown else 0 - shot = len(result.screenshot) if result.screenshot else 0 - return html, md, shot - - -def run_backend( - spec: BackendSpec, - manifest: list[dict], - config: ArchiveConfig, - pricing: Pricing, -) -> BackendRun: - run = BackendRun(name=spec.name) - if spec.precheck is not None: - reason = spec.precheck(config) - if reason: - run.skipped = reason - logger.warning("%s skipped: %s", spec.name, reason) - return run - try: - fetcher = spec.factory(config) - except Exception as e: # construction (e.g. missing key) — skip cleanly - run.skipped = f"could not construct {spec.name}: {e}" - logger.warning(run.skipped) - return run - - cm = fetcher if spec.context else nullcontext(fetcher) - try: - with cm as live: - for record in manifest: - run.rows.append(_capture_one(spec.name, live, record, pricing)) - except FetchError as e: - # A browser backend can fail to even start (e.g. cloakbrowser not - # installed). Record it as a skip rather than crashing the bake-off. - if not run.rows: - run.skipped = f"{spec.name} unavailable: {e}" - logger.warning(run.skipped) - else: - raise - return run - - -def _capture_one(backend: str, fetcher: Fetcher, record: dict, pricing: Pricing) -> Row: - url = record["url"] - category = record.get("category", "normal") - start = time.monotonic() - try: - result = fetcher.fetch(url) - except FetchError as e: - return Row( - backend, - url, - category, - False, - "fetch_error", - round(time.monotonic() - start, 2), - 0, - 0, - 0, - 0.0, - error=str(e)[:300], - ) - except Exception as e: # backend bug / unexpected SDK error - return Row( - backend, - url, - category, - False, - "exception", - round(time.monotonic() - start, 2), - 0, - 0, - 0, - 0.0, - error=str(e)[:300], - ) - - seconds = round(time.monotonic() - start, 2) - verdict = evaluate(result) - html_b, md_b, shot_b = _sizes(result) - response_bytes = html_b + shot_b - cost = ( - estimate_cost(backend, result, response_bytes, pricing) - if verdict.passed - else 0.0 - ) - return Row( - backend, - url, - category, - verdict.passed, - verdict.reason or "ok", - seconds, - html_b, - md_b, - shot_b, - round(cost, 6), - ) - - -# --- Reporting --------------------------------------------------------------- -def write_csv(path: str, runs: list[BackendRun]) -> None: - buf = io.StringIO() - w = csv.writer(buf) - w.writerow( - [ - "backend", - "url", - "category", - "passed", - "reason", - "seconds", - "html_bytes", - "md_bytes", - "screenshot_bytes", - "cost_usd", - "error", - ] - ) - for run in runs: - for r in run.rows: - w.writerow( - [ - r.backend, - r.url, - r.category, - r.passed, - r.reason, - r.seconds, - r.html_bytes, - r.md_bytes, - r.screenshot_bytes, - r.cost_usd, - r.error, - ] - ) - Path(path).write_text(buf.getvalue(), encoding="utf-8") - - -def summarize(runs: list[BackendRun], urls_per_question: int, tail_share: float) -> str: - cats = ["normal", "cloudflare", "pdf"] - lines = [] - header = ( - f"{'backend':<18}{'overall':>9}" - + "".join(f"{c:>11}" for c in cats) - + f"{'med s':>8}{'$/page':>10}{'proj $/q':>10}" - ) - lines.append(header) - lines.append("-" * len(header)) - for run in runs: - if run.skipped: - lines.append(f"{run.name:<18} SKIPPED: {run.skipped[:80]}") - continue - total = len(run.rows) - passed = [r for r in run.rows if r.passed] - overall = f"{len(passed)}/{total}" - - def cat_rate(cat: str) -> str: - rows = [r for r in run.rows if r.category == cat] - if not rows: - return "-" - ok = sum(1 for r in rows if r.passed) - return f"{ok}/{len(rows)}" - - med = statistics.median([r.seconds for r in run.rows]) if run.rows else 0 - cost_per = statistics.mean([r.cost_usd for r in passed]) if passed else 0.0 - # Illustrative: if THIS backend alone handled the whole post-Playwright - # tail of a question. (tail_share × urls × $/successful page.) - proj = tail_share * urls_per_question * cost_per - lines.append( - f"{run.name:<18}{overall:>9}" - + "".join(f"{cat_rate(c):>11}" for c in cats) - + f"{med:>8.1f}{cost_per:>10.5f}{proj:>10.3f}" - ) - note = ( - f"\nproj $/q assumes one backend covers a {tail_share:.0%} tail of " - f"{urls_per_question} URLs/question, BEFORE the TTL cache (which makes " - f"re-runs nearly free). Costs are model estimates, not billed amounts." - ) - return "\n".join(lines) + "\n" + note - - -def load_manifest(path: str | None) -> list[dict]: - if not path: - return SAMPLE_MANIFEST - records = [] - for line in Path(path).read_text(encoding="utf-8").splitlines(): - line = line.strip() - if line: - records.append(json.loads(line)) - return records - - -def main(argv: list[str] | None = None) -> int: - logging.basicConfig(level=logging.WARNING, format="%(levelname)s %(message)s") - p = argparse.ArgumentParser(description="Capture-backend bake-off.") - p.add_argument( - "--manifest", help="JSONL of {url, category}. Omit for the built-in sample." - ) - p.add_argument( - "--backends", - default="playwright,cloakbrowser,firecrawl,firecrawl-stealth,hyperbrowser,pdf", - help="Comma-separated subset of: " + ", ".join(BACKENDS), - ) - p.add_argument("--out", default="benchmark.csv", help="CSV output path.") - p.add_argument("--urls-per-question", type=int, default=450) - p.add_argument( - "--tail-share", - type=float, - default=0.30, - help="Fraction of URLs that fall through Playwright.", - ) - p.add_argument("--firecrawl-credit-usd", type=float, default=0.00083) - p.add_argument("--hyperbrowser-credit-usd", type=float, default=0.001) - args = p.parse_args(argv) - - config = ArchiveConfig.from_env() - pricing = Pricing( - firecrawl_credit_usd=args.firecrawl_credit_usd, - hyperbrowser_credit_usd=args.hyperbrowser_credit_usd, - ) - manifest = load_manifest(args.manifest) - - selected = [b.strip() for b in args.backends.split(",") if b.strip()] - unknown = [b for b in selected if b not in BACKENDS] - if unknown: - p.error(f"unknown backends: {unknown}. Choose from {list(BACKENDS)}") - - runs: list[BackendRun] = [] - for name in selected: - print(f"running {name} over {len(manifest)} URLs...", file=sys.stderr) - runs.append(run_backend(BACKENDS[name], manifest, config, pricing)) - - write_csv(args.out, runs) - print("\n" + summarize(runs, args.urls_per_question, args.tail_share)) - print(f"\nper-URL detail written to {args.out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py deleted file mode 100644 index 7d722c69..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Canonicalize URLs so trivially-different links collapse to one dedup key. - -Every capture of a page is grouped under ``url_hash`` (see :mod:`models`). -Historically that hashed the *raw* URL string, so ``…/x``, ``…/x/``, -``…/x?utm_source=…`` and ``…/x#frag`` were four different "sources" — inflating -both storage and any "how many sources have we covered" count. - -This module normalizes away differences that do **not** change *which page* you -get, so the dedup key is stable across those variants: - - - lowercase scheme + host, strip a default port (``:80`` / ``:443``) - - drop the fragment (``#…``) - - drop known analytics / click-tracking query params, then sort the rest - - normalize a trailing slash (``…/x/`` -> ``…/x``; root collapses to no path) - -It is deliberately conservative. It does **not** upgrade ``http`` -> ``https`` or -strip ``www.``: those can resolve to genuinely different pages on some hosts, so -collapsing them belongs to a later, opt-in phase (see ``ROADMAP.md``). -""" - -from __future__ import annotations - -from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit - -# Query params that are analytics/click tracking and never select the page. -# Matched case-insensitively; any key starting with a prefix below is also -# dropped. Bare ``ref`` / ``source`` are intentionally left alone — they are too -# often load-bearing (API refs, content selectors) to drop blindly. -_TRACKING_PARAMS = frozenset( - { - "gclid", - "gclsrc", - "dclid", - "gbraid", - "wbraid", - "fbclid", - "msclkid", - "yclid", - "twclid", - "mc_eid", - "mc_cid", - "_hsenc", - "_hsmi", - "igshid", - "igsh", - "vero_id", - "vero_conv", - "oly_anon_id", - "oly_enc_id", - "spm", - "scm", - "ref_src", - "ref_url", - } -) -_TRACKING_PREFIXES = ("utm_",) - -_DEFAULT_PORTS = {"http": "80", "https": "443"} - - -def _is_tracking(key: str) -> bool: - k = key.lower() - return k in _TRACKING_PARAMS or any(k.startswith(p) for p in _TRACKING_PREFIXES) - - -def canonicalize_url(url: str) -> str: - """Return a normalized form of ``url`` to use as a dedup key. - - Idempotent — ``canonicalize_url(canonicalize_url(u)) == canonicalize_url(u)``. - Non-http(s) or unparsable input is returned stripped but otherwise as-is - (e.g. ``mailto:``, relative paths), so callers can pass anything safely. - """ - if not url: - return url - raw = url.strip() - # urlsplit() itself rarely raises; .hostname/.port are LAZY properties that - # raise ValueError on junk like "http://root{--x:80/" or bad IPv6 — so the - # guard must cover the whole netloc normalization, not just the split. - try: - parts = urlsplit(raw) - if parts.scheme not in ("http", "https") or not parts.netloc: - return raw - - scheme = parts.scheme.lower() - - # netloc: lowercase host (bracket IPv6), keep userinfo, strip default - # port. - host = (parts.hostname or "").lower() - if ":" in host: # IPv6 literal - host = f"[{host}]" - netloc = host - if parts.username is not None: - auth = parts.username - if parts.password is not None: - auth += f":{parts.password}" - netloc = f"{auth}@{netloc}" - if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get(scheme): - netloc += f":{parts.port}" - except ValueError: - return raw - - # path: collapse the bare root to empty; drop a trailing slash otherwise. - path = parts.path - if path in ("", "/"): - path = "" - elif path.endswith("/"): - path = path.rstrip("/") - - # query: drop tracking params, then sort so order doesn't matter. - kept = [ - (k, v) - for k, v in parse_qsl(parts.query, keep_blank_values=True) - if not _is_tracking(k) - ] - kept.sort() - query = urlencode(kept) - - # fragment: always dropped. - return urlunsplit((scheme, netloc, path, query, "")) diff --git a/forecasting_tools/agents_and_tools/source_archive/catalog.py b/forecasting_tools/agents_and_tools/source_archive/catalog.py deleted file mode 100644 index 7712760d..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/catalog.py +++ /dev/null @@ -1,571 +0,0 @@ -"""Generate a coworker-legible catalog over the hash-addressed store. - -The content store is keyed by URL/content hash — great for dedup, opaque to a -human browsing the bucket. This builds a browsable ``catalog/`` layer on top by -joining the citation manifests (who cited what, on which question, with which -tool) with the per-URL index (what actually got captured). Blobs are never moved -or duplicated; the catalog only writes small HTML/CSV pointer pages. - -Views (question-primary, with two cross-views): - - catalog/READ_ME_FIRST.html plain-language explainer for coworkers - catalog/index.html landing page + headline counts - catalog/by-question/.html ★ the encyclopedia for one question: - catalog/by-question/.csv every source, deduped, tagged with the - bots/tools/queries that used it - catalog/by-bot/.html one bot's sources across questions - catalog/by-domain/.html sources grouped by site - -The question view is the default because that's how post-mortems think -("what sources informed question X?"); ``by-bot`` groups a bot's sources -across questions, and ``by-domain`` groups them by site. -""" - -from __future__ import annotations - -import csv -import html -import io -from collections import defaultdict -from urllib.parse import urlsplit - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive import manifest as manifest_io -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.models import ( - CitationRecord, - url_hash, -) -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) - -_UNKNOWN_Q = "unknown-question" - -# Tools that fetch data/API endpoints, not human-readable web pages. A URL only -# ever touched by one of these is a data call (e.g. a bot's run_code pulling a -# CSV), so it is kept out of the page-oriented catalog (it stays in the raw -# manifests). A URL also seen via search/page-fetch is treated as a real page. -_NON_PAGE_TOOLS = { - "run_code", - "code", - "python", - "run_python", - "code_interpreter", - "execute_code", - "bash", - "shell", -} - - -def tool_call_only(citations: list) -> bool: - """True if a URL was touched *only* by code-execution tools (a data/API call, - not a page a bot read).""" - tools = {(c.tool_name or "").lower() for c in citations} - code_tools = tools & _NON_PAGE_TOOLS - other_tools = tools - _NON_PAGE_TOOLS - {""} - return bool(code_tools) and not other_tools - - -def _is_tool_call_only(source: "Source") -> bool: - return tool_call_only(source.citations) - - -# Search-engine result pages are navigation, not sources — a bot citing a -# google/duckduckgo search URL hasn't handed us a page worth archiving. -_SEARCH_HOSTS = { - "duckduckgo.com", - "bing.com", - "search.brave.com", - "search.yahoo.com", - "ecosia.org", - "startpage.com", - "baidu.com", - "ask.com", - "qwant.com", - "search.marginalia.nu", - "kagi.com", -} -# Percent-encoded junk that means the extractor swallowed markdown / a second URL -# / control chars into the URL (legacy captures from before extraction hardening). -_MALFORMED_MARKERS = ("%5b", "%5d", "%5c", "%0a", "%0d", "%28http", "%29%5b") - - -def is_search_url(url: str) -> bool: - try: - host = urlsplit(url).netloc.lower() - except ValueError: # unparsable (see is_malformed_url) — not a search page - return False - host = host[4:] if host.startswith("www.") else host - return host in _SEARCH_HOSTS or host == "google.com" or host.startswith("google.") - - -def is_malformed_url(url: str) -> bool: - try: - urlsplit(url) - except ValueError: # e.g. a bare "http://[" — urlsplit: "Invalid IPv6 URL" - return True - low = url.lower() - return url.count("://") > 1 or any(m in low for m in _MALFORMED_MARKERS) - - -def exclusion_reason(url: str, citations: list) -> str | None: - """Why a cited URL is kept out of the page catalog / coverage, or ``None`` to - keep it. ``malformed`` (extractor junk), ``search`` (search-engine results), - ``tool_call`` (data/API endpoint touched only by code tools).""" - if is_malformed_url(url): - return "malformed" - if is_search_url(url): - return "search" - if tool_call_only(citations): - return "tool_call" - return None - - -class Citation(BaseModel): - bot: str | None = None - question_id: str | None = None - question_url: str | None = None - run_id: str | None = None - tool_name: str | None = None - origin: str | None = None - query: str | None = None - cited_url: str = "" # the original URL as cited (pre-canonicalization) - - -class Source(BaseModel): - canonical_url: str - domain: str - captured: bool = False - content_hash: str | None = None - html_key: str | None = None # store-relative (no prefix) - screenshot_key: str | None = None - markdown_key: str | None = None - citations: list[Citation] = [] - - @property - def bots(self) -> list[str]: - return sorted({c.bot for c in self.citations if c.bot}) - - @property - def question_ids(self) -> list[str]: - return sorted({c.question_id for c in self.citations if c.question_id}) - - -class CatalogData(BaseModel): - sources: list[Source] = [] - excluded: dict[str, int] = {} # exclusion reason -> count of URLs dropped - - @property - def hidden_total(self) -> int: - return sum(self.excluded.values()) - - def by_question(self) -> dict[str, list[Source]]: - out: dict[str, list[Source]] = defaultdict(list) - for s in self.sources: - qids = s.question_ids or [_UNKNOWN_Q] - for qid in qids: - out[qid].append(s) - return out - - def by_bot(self) -> dict[str, list[Source]]: - out: dict[str, list[Source]] = defaultdict(list) - for s in self.sources: - for bot in s.bots or ["(no bot)"]: - out[bot].append(s) - return out - - def by_domain(self) -> dict[str, list[Source]]: - out: dict[str, list[Source]] = defaultdict(list) - for s in self.sources: - out[s.domain].append(s) - return out - - def question_url(self, qid: str) -> str | None: - for s in self.sources: - for c in s.citations: - if c.question_id == qid and c.question_url: - return c.question_url - return None - - -# --------------------------------------------------------------------------- # -# Build (join manifests + index) -# --------------------------------------------------------------------------- # -def _domain(url: str) -> str: - try: - host = urlsplit(url).netloc.lower() - except ValueError: # malformed URL — caller falls back to "(unknown)" - return "" - return host[4:] if host.startswith("www.") else host - - -def _strip_prefix(key: str | None, prefix: str) -> str | None: - if not key: - return None - p = prefix.rstrip("/") + "/" - return key[len(p) :] if key.startswith(p) else key - - -def _latest_capture(store: ContentStore, canonical_url: str) -> dict | None: - """Return the latest stored capture dict for a URL (ignoring TTL), following - a redirect alias if present. ``None`` if nothing was ever captured.""" - index = store._read_index(url_hash(canonical_url)) - if not index: - return None - if index.get("alias_of"): - index = store._read_index(index["alias_of"]) - if not index: - return None - ch = index.get("latest_content_hash") - return (index.get("captures") or {}).get(ch) - - -def _load_all_records(store: BlobStore, prefix: str) -> list[CitationRecord]: - records: list[CitationRecord] = [] - for key in store.list_keys(f"{prefix.rstrip('/')}/manifests/"): - if not key.endswith(".jsonl"): - continue - try: - records.extend(manifest_io.loads(store.get(key).decode("utf-8"))) - except (UnicodeDecodeError, ValueError): - continue - return records - - -def build_sources(store: BlobStore, config: ArchiveConfig) -> list[Source]: - """Join every manifest with the index into one ``Source`` per canonical URL. - - Unfiltered (includes tool/API-call URLs) so other tools — e.g. the coverage - report — can classify them. The catalog itself filters these out. - """ - prefix = config.s3_prefix.rstrip("/") - cstore = ContentStore(store, config) - records = _load_all_records(store, prefix) - - grouped: dict[str, list[CitationRecord]] = defaultdict(list) - for r in records: - if r.url: - grouped[canonicalize_url(r.url)].append(r) - - sources: list[Source] = [] - for canonical, recs in sorted(grouped.items()): - cap = _latest_capture(cstore, canonical) - source = Source( - canonical_url=canonical, - domain=_domain(canonical) or "(unknown)", - captured=cap is not None, - content_hash=(cap or {}).get("content_hash"), - html_key=_strip_prefix((cap or {}).get("html_key"), prefix), - screenshot_key=_strip_prefix((cap or {}).get("screenshot_key"), prefix), - markdown_key=_strip_prefix((cap or {}).get("markdown_key"), prefix), - citations=[ - Citation( - bot=r.bot, - question_id=r.question_id or r.metaculus_id, - question_url=r.question_url, - run_id=r.run_id, - tool_name=r.tool_name, - origin=r.origin, - query=r.query, - cited_url=r.url, - ) - for r in recs - ], - ) - sources.append(source) - return sources - - -def build_catalog(store: BlobStore, config: ArchiveConfig) -> CatalogData: - sources = build_sources(store, config) - pages: list[Source] = [] - excluded: dict[str, int] = defaultdict(int) - for s in sources: - reason = exclusion_reason(s.canonical_url, s.citations) - if reason: - excluded[reason] += 1 - else: - pages.append(s) - return CatalogData(sources=pages, excluded=dict(excluded)) - - -# --------------------------------------------------------------------------- # -# Render -# --------------------------------------------------------------------------- # -_CSS = """ -body{font:14px/1.5 -apple-system,Segoe UI,Roboto,sans-serif;margin:0;color:#1a1a1a;background:#fafafa} -header{background:#1f2937;color:#fff;padding:16px 24px} -header a{color:#cbd5e1} -h1{font-size:20px;margin:0 0 4px} -.wrap{padding:24px;max-width:1100px;margin:0 auto} -.muted{color:#6b7280} -.badge{display:inline-block;font-size:11px;padding:1px 7px;border-radius:10px} -.ok{background:#dcfce7;color:#166534}.no{background:#fee2e2;color:#991b1b} -.card{background:#fff;border:1px solid #e5e7eb;border-radius:8px;padding:12px;margin:12px 0;display:flex;gap:12px} -.card img{width:160px;height:110px;object-fit:cover;object-position:top;border:1px solid #e5e7eb;border-radius:4px;background:#f3f4f6} -.card .meta{flex:1;min-width:0} -.card .u{font-weight:600;word-break:break-all} -.tags{margin-top:6px} -.tag{display:inline-block;background:#eef2ff;color:#3730a3;font-size:11px;padding:1px 7px;border-radius:10px;margin:2px 4px 2px 0} -.links a{margin-right:10px;font-size:12px} -table{border-collapse:collapse;width:100%;background:#fff} -td,th{border:1px solid #e5e7eb;padding:6px 8px;text-align:left;font-size:13px} -th{background:#f3f4f6} -a.grid{display:inline-block;margin:4px 12px 4px 0} -""" - - -def _esc(s) -> str: - return html.escape(str(s)) if s is not None else "" - - -def _page(title: str, body: str, rel_root: str) -> str: - return ( - "" - f"{_esc(title)}" - f"

Source Archive

" - f"← catalog home
" - f"
{body}
" - ) - - -class Linker: - """Turns a store-relative blob key into a link a coworker can open.""" - - def __init__(self, store: BlobStore, config: ArchiveConfig): - from forecasting_tools.agents_and_tools.source_archive.storage import ( - S3BlobStore, - ) - - self.is_s3 = isinstance(store, S3BlobStore) - self.bucket = config.s3_bucket - self.region = config.aws_region - self.prefix = config.s3_prefix.rstrip("/") - - def url(self, rel_key: str | None, rel_root: str) -> str | None: - if not rel_key: - return None - if self.is_s3: - host = ( - f"{self.bucket}.s3.{self.region}.amazonaws.com" - if self.region - else f"{self.bucket}.s3.amazonaws.com" - ) - return f"https://{host}/{self.prefix}/{rel_key}" - return f"{rel_root}{rel_key}" # local: relative within the prefix dir - - -def _source_card(s: Source, linker: Linker, rel_root: str) -> str: - shot = linker.url(s.screenshot_key, rel_root) - html_link = linker.url(s.html_key, rel_root) - md_link = linker.url(s.markdown_key, rel_root) - badge = ( - "captured" - if s.captured - else "not captured" - ) - img = ( - f"screenshot" - if shot - else "
" - ) - tools = sorted({c.tool_name for c in s.citations if c.tool_name}) - tags = "".join(f"{_esc(b)}" for b in s.bots) - tool_tags = "".join(f"{_esc(t)}" for t in tools) - links = [] - if html_link: - links.append(f"HTML") - if md_link: - links.append(f"markdown") - if shot: - links.append(f"screenshot") - links.append(f"live ↗") - return ( - f"
{img}
" - f"
{_esc(s.canonical_url)}
" - f"
{_esc(s.domain)} · {badge}
" - f"
{tags}{tool_tags}
" - f"
{' '.join(links)}
" - f"
" - ) - - -def _question_csv(sources: list[Source]) -> str: - buf = io.StringIO() - w = csv.writer(buf) - w.writerow(["url", "domain", "captured", "bots", "tools", "screenshot_key"]) - for s in sources: - tools = sorted({c.tool_name for c in s.citations if c.tool_name}) - w.writerow( - [ - s.canonical_url, - s.domain, - "yes" if s.captured else "no", - "; ".join(s.bots), - "; ".join(tools), - s.screenshot_key or "", - ] - ) - return buf.getvalue() - - -# --------------------------------------------------------------------------- # -# Write -# --------------------------------------------------------------------------- # -class CatalogSummary(BaseModel): - sources: int = 0 - captured: int = 0 - questions: int = 0 - bots: int = 0 - domains: int = 0 - excluded: dict[str, int] = {} - - def __str__(self) -> str: - excl = sum(self.excluded.values()) - breakdown = ( - " (" + ", ".join(f"{k}={v}" for k, v in sorted(self.excluded.items())) + ")" - if self.excluded - else "" - ) - return ( - f"Catalog: {self.sources} page sources ({self.captured} captured) across " - f"{self.questions} questions, {self.bots} bots, {self.domains} domains " - f"— {excl} non-page URLs excluded{breakdown}" - ) - - -def _slug(value: str) -> str: - # Keep dots so domains stay readable (a.test.html); collapse anything else. - keep = [c if c.isalnum() or c in "-_." else "-" for c in value] - out = "".join(keep).strip("-.").replace("..", ".")[:80] - return out or "x" - - -def write_catalog( - store: BlobStore, - config: ArchiveConfig, - out_store: BlobStore | None = None, -) -> CatalogSummary: - """Build the catalog from ``store`` and write it to ``out_store`` (default: - ``store``). Pass a separate ``out_store`` to preview a live bucket's catalog - into a local directory without mutating the bucket.""" - prefix = config.s3_prefix.rstrip("/") - data = build_catalog(store, config) - out = out_store or store - linker = Linker(out, config) - - def put(rel: str, body: str, ctype: str) -> None: - out.put(f"{prefix}/catalog/{rel}", body.encode("utf-8"), content_type=ctype) - - by_q = data.by_question() - by_b = data.by_bot() - by_d = data.by_domain() - - # Per-question pages (the encyclopedia) + CSVs. rel_root: catalog// -> ../../ - rr2 = "../../" - for qid, sources in sorted(by_q.items()): - sources = sorted(sources, key=lambda s: s.canonical_url) - qurl = data.question_url(qid) - head = f"

Question {_esc(qid)}

" - if qurl: - head += f"

{_esc(qurl)} ↗

" - head += ( - f"

{len(sources)} source(s); " - f"{sum(s.captured for s in sources)} captured · " - f"download CSV

" - ) - cards = "".join(_source_card(s, linker, rr2) for s in sources) - put( - f"by-question/{_slug(qid)}.html", - _page(f"Question {qid}", head + cards, rr2), - "text/html", - ) - put(f"by-question/{_slug(qid)}.csv", _question_csv(sources), "text/csv") - - # Per-bot and per-domain cross-views. - for bot, sources in sorted(by_b.items()): - sources = sorted(sources, key=lambda s: s.canonical_url) - body = f"

Bot: {_esc(bot)}

{len(sources)} source(s)

" - body += "".join(_source_card(s, linker, rr2) for s in sources) - put(f"by-bot/{_slug(bot)}.html", _page(f"Bot {bot}", body, rr2), "text/html") - - for domain, sources in sorted(by_d.items()): - sources = sorted(sources, key=lambda s: s.canonical_url) - body = f"

Site: {_esc(domain)}

{len(sources)} source(s)

" - body += "".join(_source_card(s, linker, rr2) for s in sources) - put( - f"by-domain/{_slug(domain)}.html", - _page(f"Site {domain}", body, rr2), - "text/html", - ) - - # Landing + readme. rel_root: catalog/ -> ../ - rr1 = "../" - index_body = _index_body(data, by_q, by_b, by_d) - put("index.html", _page("Catalog", index_body, rr1), "text/html") - put("READ_ME_FIRST.html", _page("Read me first", _readme_body(), rr1), "text/html") - - return CatalogSummary( - sources=len(data.sources), - captured=sum(s.captured for s in data.sources), - questions=len(by_q), - bots=len(by_b), - domains=len(by_d), - excluded=data.excluded, - ) - - -def _index_body(data, by_q, by_b, by_d) -> str: - captured = sum(s.captured for s in data.sources) - - def links(items: dict, view: str) -> str: - rows = [] - for key, sources in sorted(items.items(), key=lambda kv: (-len(kv[1]), kv[0])): - rows.append( - f"" - f"{_esc(key)} ({len(sources)})" - ) - return "".join(rows) - - hidden_note = ( - f" · {data.hidden_total} non-page URLs hidden " - f"({', '.join(f'{k} {v}' for k, v in sorted(data.excluded.items()))})" - if data.hidden_total - else "" - ) - return ( - f"

What is this? →

" - f"

{len(data.sources)} page sources ({captured} captured) · " - f"{len(by_q)} questions · {len(by_b)} bots · {len(by_d)} sites{hidden_note}

" - f"

By question

The encyclopedia of sources per " - f"question — start here.

{links(by_q, 'by-question')}" - f"

By bot

{links(by_b, 'by-bot')}" - f"

By site

{links(by_d, 'by-domain')}" - ) - - -def _readme_body() -> str: - return ( - "

What is this bucket?

" - "

This is a source archive: for every web page a forecasting bot " - "cited, we save a snapshot — the page's HTML, a full-page " - "screenshot, and a clean markdown copy — so a forecast can be " - "audited later even if the original page changes or disappears.

" - "

How to browse it

" - "
    " - "
  • Open index.html (the catalog home).
    • " - "
  • By question is the main view: pick a question to see every " - "source used for it, who used it, and a screenshot of each.
    • " - "
  • By bot shows one bot's sources across questions; By site " - "groups sources by website.
    • " - "
  • Each question also has a CSV you can open in a spreadsheet.
    • " - "
" - "

The folders with long hash names (content/, index/) are " - "the machine-readable store — you don't need to open those.

" - ) diff --git a/forecasting_tools/agents_and_tools/source_archive/cli.py b/forecasting_tools/agents_and_tools/source_archive/cli.py deleted file mode 100644 index 0a93a1f3..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/cli.py +++ /dev/null @@ -1,353 +0,0 @@ -"""Command-line interface for the source archive. - - # See the resolved configuration (secrets masked) - python -m forecasting_tools.agents_and_tools.source_archive.cli check - - # Capture every URL in a manifest and upload to S3 (uses WEB_ARCHIVE_S3_BUCKET) - python -m forecasting_tools.agents_and_tools.source_archive.cli capture run.jsonl - - # Same, but store to a local folder instead of S3 (no AWS needed) - python -m forecasting_tools.agents_and_tools.source_archive.cli capture run.jsonl --local ./archive - -If installed via the ``source-archive`` extra, the ``source-archive`` console -command is equivalent to ``python -m ...cli``. -""" - -from __future__ import annotations - -import argparse -import sys - -from forecasting_tools.agents_and_tools.source_archive import manifest as manifest_io -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.fetchers import ( - build_default_fetcher, -) - - -def _load_dotenv() -> None: - try: - from dotenv import load_dotenv - - load_dotenv() - except ImportError: - pass - - -def _mask(value: str | None) -> str: - if not value: - return "(unset)" - if len(value) <= 6: - return "***" - return f"{value[:3]}…{value[-2:]}" - - -def _make_blob_store(config: ArchiveConfig, local_dir: str | None, bucket: str | None): - if local_dir: - from forecasting_tools.agents_and_tools.source_archive.storage import ( - LocalBlobStore, - ) - - return LocalBlobStore(local_dir) - bucket = bucket or config.s3_bucket - if not bucket: - sys.exit( - "No S3 bucket configured. Set WEB_ARCHIVE_S3_BUCKET (or pass --bucket), " - "or use --local DIR to store to the filesystem." - ) - from forecasting_tools.agents_and_tools.source_archive.storage import S3BlobStore - - return S3BlobStore(bucket, config=config) - - -def _cmd_check(config: ArchiveConfig) -> int: - print("Source-archive configuration (secrets masked):") - print(f" S3 bucket : {config.s3_bucket or '(unset)'}") - print(f" S3 prefix : {config.s3_prefix}") - print(f" AWS profile : {config.aws_profile or '(default chain)'}") - print(f" AWS region : {config.aws_region or '(default)'}") - print(f" Firecrawl API key : {_mask(config.firecrawl_api_key)}") - print(f" Firecrawl proxy mode : {config.firecrawl_proxy}") - print(f" Hyperbrowser API key : {_mask(config.hyperbrowser_api_key)}") - print(f" Hyperbrowser proxy : {config.hyperbrowser_use_proxy}") - print(f" CloakBrowser module : {config.cloakbrowser_import}") - print(f" PDF max pages : {config.pdf_max_pages}") - print(f" TTL (days) : {config.ttl_days}") - print(f" Screenshot format : {config.screenshot_format}") - print(f" Screenshot max height: {config.screenshot_max_height}") - return 0 - - -def _cmd_capture(args, config: ArchiveConfig) -> int: - from forecasting_tools.agents_and_tools.source_archive.manifest import unique_urls - from forecasting_tools.agents_and_tools.source_archive.pipeline import ( - capture_urls_concurrent, - ) - - records = manifest_io.read_file(args.manifest) - - overrides = {} - if getattr(args, "no_hyperbrowser", False): - overrides["hyperbrowser_api_key"] = None - if getattr(args, "concurrency", None): - overrides["concurrency"] = args.concurrency - if overrides: - config = config.model_copy(update=overrides) - if "hyperbrowser_api_key" in overrides: - print("Hyperbrowser fallback DISABLED for this run.") - - store = ContentStore(_make_blob_store(config, args.local, args.bucket), config) - - urls = list(unique_urls(records)) - if args.limit: - urls = urls[: args.limit] - target = args.local or f"s3://{args.bucket or config.s3_bucket}/{config.s3_prefix}" - print( - f"Capturing {len(urls)} URL(s) at concurrency {config.concurrency} -> {target}" - ) - - summary = capture_urls_concurrent(urls, store, config, build_default_fetcher) - print(summary) - - from forecasting_tools.agents_and_tools.source_archive import cost as cost_mod - - run_cost = cost_mod.estimate_run_cost(summary, config, run_id=args.run_id) - print(run_cost) - - run_id = args.run_id or (records[0].run_id if records else None) - if run_id: - from forecasting_tools.agents_and_tools.source_archive import reports - - key = reports.write_run_report( - store.blobs, run_id, summary, config, group=args.group - ) - print(f"Wrote run outcomes -> {key}") - key = cost_mod.write_cost_report( - store.blobs, run_id, run_cost, config, group=args.group - ) - print(f"Wrote cost report -> {key}") - - # Failures leave no cache entry, so re-running retries exactly them. Write a - # retry manifest (with provenance) so coming back — e.g. with hyperbrowser - # re-enabled — is one command over only the sites that still need it. - failed = { - o.url for o in summary.outcomes if o.status in ("quality_failed", "error") - } - if failed: - from forecasting_tools.agents_and_tools.source_archive.ingest import ( - dedupe_records, - ) - - retry_records = dedupe_records(r for r in records if r.url in failed) - retry_path = args.retry_out or f"{run_id or 'run'}_needs_retry.jsonl" - manifest_io.write_file(retry_path, retry_records) - print( - f"{len(failed)} URL(s) failed -> retry manifest {retry_path}\n" - f" come back later with: source-archive capture {retry_path} " - f"--run-id {run_id or ''} (hyperbrowser on by default)" - ) - - if args.upload_manifest: - if not run_id: - sys.exit("--upload-manifest needs --run-id (no run_id found in records)") - manifest_io.write_blob(store.blobs, run_id, records, config, group=args.group) - print( - f"Uploaded manifest -> {manifest_io.manifest_key(run_id, config, args.group)}" - ) - return 0 - - -def _cmd_ingest_traces(args, config: ArchiveConfig) -> int: - from forecasting_tools.agents_and_tools.source_archive.ingest import ( - dedupe_records, - harvest_run, - ) - - run_id = args.run_id # None -> derived from the run dir name - records = harvest_run(args.run_dir, run_id=run_id, bot=args.bot) - if args.dedupe: - records = dedupe_records(records) - run_id = run_id or (records[0].run_id if records else None) - print(f"Ingested {len(records)} citation record(s) from traces in {args.run_dir}") - - out_path = args.out or f"{run_id or 'traces'}.jsonl" - if not args.upload or args.out: - manifest_io.write_file(out_path, records) - print(f"Wrote manifest -> {out_path}") - if args.upload: - if not run_id: - sys.exit("--upload needs a run id (pass --run-id; none found in records)") - store = _make_blob_store(config, None, args.bucket) - manifest_io.write_blob(store, run_id, records, config, group=args.group) - print( - f"Uploaded manifest -> {manifest_io.manifest_key(run_id, config, args.group)}" - ) - return 0 - - -def _cmd_catalog(args, config: ArchiveConfig) -> int: - from forecasting_tools.agents_and_tools.source_archive.catalog import write_catalog - - store = _make_blob_store(config, args.local, args.bucket) - target = args.local or f"s3://{args.bucket or config.s3_bucket}/{config.s3_prefix}" - print(f"Building catalog from manifests + index -> {target}/catalog/") - summary = write_catalog(store, config) - print(summary) - print(f"Open {config.s3_prefix}/catalog/index.html") - return 0 - - -def _cmd_coverage(args, config: ArchiveConfig) -> int: - from pathlib import Path - - from forecasting_tools.agents_and_tools.source_archive import reports - from forecasting_tools.agents_and_tools.source_archive.catalog import build_sources - from forecasting_tools.agents_and_tools.source_archive.coverage import ( - MODES, - coverage_from_sources, - ) - - store = _make_blob_store(config, args.local, args.bucket) - sources = build_sources(store, config) # read manifests + index once - outcomes = reports.read_outcomes(store, config) or None - modes = MODES if args.mode == "both" else (args.mode,) - for mode in modes: - report = coverage_from_sources(sources, mode, outcomes) - print(report) - print() - if args.csv: - Path(f"{args.csv}_{mode}.csv").write_text(report.to_csv()) - print(f"Wrote {args.csv}_{mode}.csv") - if report.missing_urls: - Path(f"{args.csv}_{mode}_missing.txt").write_text( - "\n".join(report.missing_urls) - ) - print(f"Wrote {args.csv}_{mode}_missing.txt") - return 0 - - -def main(argv: list[str] | None = None) -> int: - _load_dotenv() - parser = argparse.ArgumentParser( - prog="source-archive", - description="Capture HTML + screenshot + markdown for the URLs a " - "forecasting bot cited, and store them with provenance.", - ) - sub = parser.add_subparsers(dest="command", required=True) - - sub.add_parser("check", help="print the resolved configuration (secrets masked)") - - cap = sub.add_parser("capture", help="capture all URLs in a citation manifest") - cap.add_argument("manifest", help="path to a citation manifest (.jsonl)") - cap.add_argument( - "--local", metavar="DIR", help="store to this directory instead of S3" - ) - cap.add_argument( - "--bucket", help="override the S3 bucket (default: WEB_ARCHIVE_S3_BUCKET)" - ) - cap.add_argument( - "--upload-manifest", - action="store_true", - help="also upload the manifest itself to manifests/.jsonl", - ) - cap.add_argument("--run-id", help="run id for the uploaded manifest") - cap.add_argument( - "--group", - help="nest the uploaded manifest/reports under this folder, e.g. " - "'sprints/myrun' (default: daily// for daily-YYYY-MM-DD " - "run ids, else adhoc/)", - ) - cap.add_argument( - "--no-hyperbrowser", - action="store_true", - help="disable the Hyperbrowser fallback for this run (others still run)", - ) - cap.add_argument( - "--retry-out", - metavar="FILE", - help="where to write the failed-URL retry manifest " - "(default: _needs_retry.jsonl)", - ) - cap.add_argument( - "--concurrency", - type=int, - metavar="N", - help="parallel browser workers for this run (overrides WEB_ARCHIVE_CONCURRENCY)", - ) - cap.add_argument( - "--limit", - type=int, - metavar="N", - help="capture only the first N URLs (chunk a big manifest; resume via cache)", - ) - - ing = sub.add_parser( - "ingest-traces", - help="build a manifest from a traced bot run directory (bot_*/q_*/traces_*.jsonl)", - ) - ing.add_argument("run_dir", help="path to a traced run directory") - ing.add_argument( - "--out", metavar="FILE", help="write the manifest to this .jsonl file" - ) - ing.add_argument("--run-id", help="run id (default: the run dir's name)") - ing.add_argument( - "--bot", - help="bot name for a flat (no bot_*/) layout (default: the run dir's name)", - ) - ing.add_argument( - "--dedupe", action="store_true", help="keep one record per URL (first seen)" - ) - ing.add_argument( - "--upload", action="store_true", help="upload the manifest to S3 manifests/" - ) - ing.add_argument( - "--group", - help="nest the uploaded manifest under this folder, e.g. 'sprints/myrun' " - "(default: daily// for daily-YYYY-MM-DD run ids, else adhoc/)", - ) - ing.add_argument("--bucket", help="override the S3 bucket") - - cat = sub.add_parser( - "catalog", - help="generate a coworker-legible HTML/CSV catalog (by question/bot/site)", - ) - cat.add_argument( - "--local", metavar="DIR", help="read/write the catalog in this directory" - ) - cat.add_argument("--bucket", help="override the S3 bucket") - - cov = sub.add_parser( - "coverage", - help="report what %% of cited sources were archived (trace vs comments)", - ) - cov.add_argument( - "--mode", - choices=["trace", "comments", "both"], - default="both", - help="which report(s) to print (default: both)", - ) - cov.add_argument( - "--csv", metavar="PREFIX", help="write PREFIX_.csv (+ _missing.txt)" - ) - cov.add_argument("--local", metavar="DIR", help="read from this directory") - cov.add_argument("--bucket", help="override the S3 bucket") - - args = parser.parse_args(argv) - config = ArchiveConfig.from_env() - - if args.command == "check": - return _cmd_check(config) - if args.command == "capture": - return _cmd_capture(args, config) - if args.command == "ingest-traces": - return _cmd_ingest_traces(args, config) - if args.command == "catalog": - return _cmd_catalog(args, config) - if args.command == "coverage": - return _cmd_coverage(args, config) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/forecasting_tools/agents_and_tools/source_archive/config.py b/forecasting_tools/agents_and_tools/source_archive/config.py deleted file mode 100644 index cfb643ef..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/config.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Configuration for the source archive, read from environment variables. - -No bucket names, credentials, or other deployment-specific values are baked in -here, so this module is safe to publish. Operators set the bucket via -``WEB_ARCHIVE_S3_BUCKET`` (see ``.env.template``). -""" - -from __future__ import annotations - -import os - -from pydantic import BaseModel - - -def _get_int(name: str, default: int) -> int: - raw = os.environ.get(name) - if raw is None or raw == "": - return default - return int(raw) - - -def _get_bool(name: str, default: bool) -> bool: - raw = os.environ.get(name) - if raw is None or raw == "": - return default - return raw.strip().lower() in ("1", "true", "yes", "on") - - -class ArchiveConfig(BaseModel): - """Runtime configuration. Construct directly in tests, or ``from_env()``.""" - - s3_bucket: str | None = None - s3_prefix: str = "source-archive" - # Local archive directory. When set, the viewer reads captures from here - # instead of S3 — handy for inspecting a local capture run with no AWS. - local_dir: str | None = None - aws_profile: str | None = None - aws_region: str | None = None - firecrawl_api_key: str | None = None - # Firecrawl proxy mode for the anti-bot path: "basic" (1 credit) | "auto" - # (1 credit, escalates to 5 on fallback) | "stealth"/"enhanced" (5 credits). - # Only the fallback Firecrawl tier pays this; basic is the default. - firecrawl_proxy: str = "basic" - hyperbrowser_api_key: str | None = None - # Hyperbrowser session knobs for the anti-bot path. Proxy turns a 1-credit - # scrape into a 10-credit one, so leave it on only for the Cloudflare tier. - hyperbrowser_use_proxy: bool = True - hyperbrowser_use_stealth: bool = True - hyperbrowser_solve_captchas: bool = True - # CloakBrowser exposes ``cloakbrowser.launch() -> Browser``; the module name - # is overridable in case the package is renamed. - cloakbrowser_import: str = "cloakbrowser" - pdf_max_pages: int = 50 # cap PDF parsing so a huge report can't blow latency/cost - ttl_days: int = 14 - screenshot_format: str = "webp" # webp | jpeg | png - screenshot_max_height: int = 16_000 # px; safety cap (under WebP's 16383 limit) - nav_timeout_ms: int = 30_000 - concurrency: int = 5 - - @classmethod - def from_env(cls) -> "ArchiveConfig": - return cls( - s3_bucket=os.environ.get("WEB_ARCHIVE_S3_BUCKET"), - s3_prefix=os.environ.get("WEB_ARCHIVE_S3_PREFIX", "source-archive"), - local_dir=os.environ.get("WEB_ARCHIVE_LOCAL_DIR"), - aws_profile=os.environ.get("WEB_ARCHIVE_AWS_PROFILE"), - aws_region=os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION"), - firecrawl_api_key=os.environ.get("FIRECRAWL_API_KEY"), - firecrawl_proxy=os.environ.get("WEB_ARCHIVE_FIRECRAWL_PROXY", "basic"), - hyperbrowser_api_key=os.environ.get("HYPERBROWSER_API_KEY"), - hyperbrowser_use_proxy=_get_bool("WEB_ARCHIVE_HYPERBROWSER_PROXY", True), - hyperbrowser_use_stealth=_get_bool( - "WEB_ARCHIVE_HYPERBROWSER_STEALTH", True - ), - hyperbrowser_solve_captchas=_get_bool( - "WEB_ARCHIVE_HYPERBROWSER_CAPTCHA", True - ), - cloakbrowser_import=os.environ.get( - "WEB_ARCHIVE_CLOAKBROWSER_IMPORT", "cloakbrowser" - ), - pdf_max_pages=_get_int("WEB_ARCHIVE_PDF_MAX_PAGES", 50), - ttl_days=_get_int("WEB_ARCHIVE_TTL_DAYS", 14), - screenshot_format=os.environ.get("WEB_ARCHIVE_SCREENSHOT_FORMAT", "webp"), - screenshot_max_height=_get_int("WEB_ARCHIVE_SCREENSHOT_MAX_HEIGHT", 16_000), - nav_timeout_ms=_get_int("WEB_ARCHIVE_NAV_TIMEOUT_MS", 30_000), - concurrency=_get_int("WEB_ARCHIVE_CONCURRENCY", 5), - ) diff --git a/forecasting_tools/agents_and_tools/source_archive/content_store.py b/forecasting_tools/agents_and_tools/source_archive/content_store.py deleted file mode 100644 index 2c0827cb..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/content_store.py +++ /dev/null @@ -1,323 +0,0 @@ -"""URL content store, keyed by URL + content hash, with a TTL cache. - -The big cost lever is **not re-fetching** a URL captured recently: a bot -re-forecasts the same open question every 20-30 minutes for weeks, citing the -same pages over and over, so temporal overlap is near-total. - - - :meth:`ContentStore.lookup` — if a URL was captured within the TTL, return - the pointer and skip the fetch entirely (the cheap path that makes re-runs - nearly free). - - :meth:`ContentStore.store` — write blobs under - ``content//.*``; if that exact content hash is - already stored, skip the write (dedup identical re-fetches) and just refresh - timestamps. - -**Redirect aliasing.** A capture is keyed by its *final* URL (after redirects), -so a link shortener (``bit.ly/x``) and the page it resolves to collapse onto one -capture instead of two. The original cited URL gets a tiny **alias index** that -points at the final URL's index, and the final URL's index lists its aliases for -provenance. So ``lookup(bit.ly/x)`` and ``lookup(final)`` both hit the same -stored page, and we never store the destination twice. - -**Cross-URL content dedup.** Different URLs that return byte-identical content -share the blobs rather than storing them three times each. The first URL to -store a given content owns the blobs; later URLs get a capture whose blob keys -point back at them and whose ``content_alias_of`` names the owner. A reverse -``index/by-content/.json`` tracks the owner and every member URL. - -Object layout (under ``config.s3_prefix``):: - - index/.json canonical: capture history + "aliases" - index/.json alias: {"alias_of": } - index/by-content/.json reverse: owner + member urls - content//.html - content//. - content//.md -""" - -from __future__ import annotations - -import json -import threading -from datetime import datetime, timedelta, timezone - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.models import ( - CaptureResult, - StoredCapture, - url_hash, - utcnow_iso, -) -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) - -_IMG_EXT = {"image/jpeg": "jpg", "image/png": "png", "image/webp": "webp"} - - -class StoreResult(BaseModel): - capture: StoredCapture - created: bool # False when the content hash was already stored (deduped) - - -def _parse_iso(ts: str) -> datetime: - dt = datetime.fromisoformat(ts) - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - return dt - - -def _capture_is_complete(cap: dict) -> bool: - """Whether a stored capture has every format we expect for its type. - - A browser capture is complete only with html + markdown + screenshot; a PDF - (which has no screenshot) only needs its markdown. Used by :meth:`lookup` so - an incomplete capture is re-fetched rather than treated as already done. - """ - if (cap.get("fetcher") or "").lower() == "pdf": - return bool(cap.get("markdown_key")) - return bool( - cap.get("html_key") and cap.get("markdown_key") and cap.get("screenshot_key") - ) - - -class ContentStore: - def __init__(self, blob_store: BlobStore, config: ArchiveConfig | None = None): - self.blobs = blob_store - self.config = config or ArchiveConfig() - self.prefix = self.config.s3_prefix.rstrip("/") - # Serializes the shared by-content reverse index across capture threads - # (concurrent runs). Per-URL index files are written by a single thread - # each, so they don't need it; the by-content index can be contended when - # different URLs return identical content. - self._content_lock = threading.Lock() - - # --- key helpers ------------------------------------------------------- - def _index_key(self, uh: str) -> str: - return f"{self.prefix}/index/{uh}.json" - - def _content_key(self, uh: str, ch: str, ext: str) -> str: - return f"{self.prefix}/content/{uh}/{ch}.{ext}" - - def _content_index_key(self, ch: str) -> str: - return f"{self.prefix}/index/by-content/{ch}.json" - - # --- index io ---------------------------------------------------------- - def _read_index(self, uh: str) -> dict | None: - key = self._index_key(uh) - if not self.blobs.exists(key): - return None - return json.loads(self.blobs.get(key).decode("utf-8")) - - def _write_index(self, uh: str, index: dict) -> None: - data = json.dumps(index, indent=2, sort_keys=True).encode("utf-8") - self.blobs.put(self._index_key(uh), data, content_type="application/json") - - def _read_content_index(self, ch: str) -> dict | None: - key = self._content_index_key(ch) - if not self.blobs.exists(key): - return None - try: - return json.loads(self.blobs.get(key).decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError): - # A concurrent writer may have left a partial local file mid-write; - # treat as absent rather than crash. The locked path below is authoritative. - return None - - def _register_content( - self, ch: str, uh: str, url: str, blob_keys: dict | None - ) -> None: - """Record that ``uh`` produced content ``ch`` in the reverse index. - - The first URL to store a given content becomes its ``canonical_url_hash`` - and owns the blob keys; later URLs with identical content are added as - ``members`` and reuse those blobs (see :meth:`store`). Locked so concurrent - capture threads with identical content don't clobber each other's members. - """ - with self._content_lock: - reverse = self._read_content_index(ch) - if reverse is None: - reverse = { - "content_hash": ch, - "canonical_url_hash": uh, - "blob_keys": blob_keys or {}, - "members": [], - } - members = reverse.setdefault("members", []) - if not any(m.get("url_hash") == uh for m in members): - members.append({"url_hash": uh, "url": url}) - data = json.dumps(reverse, indent=2, sort_keys=True).encode("utf-8") - self.blobs.put( - self._content_index_key(ch), data, content_type="application/json" - ) - - # --- public api -------------------------------------------------------- - def lookup(self, url: str) -> StoredCapture | None: - """Return the latest stored capture if within the TTL, else ``None``. - - A non-``None`` return means callers can skip fetching this URL. If ``url`` - is an alias of a previously-redirected target, the alias is followed to - the canonical capture. - """ - uh = url_hash(url) - index = self._read_index(uh) - if not index: - return None - alias_of = index.get("alias_of") - if alias_of: # follow the alias to the canonical (final-URL) index - index = self._read_index(alias_of) - if not index: - return None - latest_ch = index.get("latest_content_hash") - captures = index.get("captures", {}) - latest = captures.get(latest_ch) - if not latest: - return None - - last_seen = _parse_iso(latest["last_seen"]) - age = datetime.now(timezone.utc) - last_seen - if age > timedelta(days=self.config.ttl_days): - return None - # Skip only a COMPLETE capture. A partial one (e.g. a failed screenshot - # encode left screenshot_key=None) is treated as a miss so the next run - # retries the missing format instead of skipping it forever. PDFs have no - # screenshot by nature, so they only need their markdown. - if not _capture_is_complete(latest): - return None - return StoredCapture.model_validate(latest) - - def store(self, result: CaptureResult) -> StoreResult: - """Persist a capture, deduping by content hash. Always updates the index. - - The capture is keyed by the *final* URL (after redirects). If the cited - URL differs from the final one, an alias index is written so the cited - URL still resolves here, and the cited URL is recorded under the - canonical index's ``aliases``. - """ - final_url = result.final_url or result.url - uh = url_hash(final_url) - ch = result.content_hash - now = utcnow_iso() - - index = self._read_index(uh) or { - "url": final_url, - "url_hash": uh, - "first_seen": now, - "captures": {}, - } - captures = index.setdefault("captures", {}) - existing = captures.get(ch) - - created = existing is None - if existing is not None: - # Identical content already stored for THIS url — skip writes, touch. - existing["last_seen"] = now - stored = StoredCapture.model_validate(existing) - else: - reverse = self._read_content_index(ch) - reuse = bool( - reverse and reverse.get("canonical_url_hash") not in (None, uh) - ) - if reuse: - # Byte-identical content already stored under a DIFFERENT url — - # point at its blobs instead of writing three more (cross-URL - # content dedup); each url still keeps its own index history. - bk = reverse.get("blob_keys", {}) - html_key = bk.get("html") - markdown_key = bk.get("markdown") - screenshot_key = bk.get("screenshot") - content_alias_of = reverse["canonical_url_hash"] - else: - html_key = screenshot_key = markdown_key = None - if result.html is not None: - html_key = self._content_key(uh, ch, "html") - self.blobs.put( - html_key, result.html.encode("utf-8"), content_type="text/html" - ) - if result.markdown is not None: - markdown_key = self._content_key(uh, ch, "md") - self.blobs.put( - markdown_key, - result.markdown.encode("utf-8"), - content_type="text/markdown", - ) - if result.screenshot is not None: - ext = _IMG_EXT.get(result.screenshot_content_type or "", "png") - screenshot_key = self._content_key(uh, ch, ext) - self.blobs.put( - screenshot_key, - result.screenshot, - content_type=result.screenshot_content_type, - ) - content_alias_of = None - stored = StoredCapture( - url=final_url, - url_hash=uh, - content_hash=ch, - status_code=result.status_code, - fetcher=result.fetcher, - captured_at=result.fetched_at, - html_key=html_key, - screenshot_key=screenshot_key, - markdown_key=markdown_key, - content_alias_of=content_alias_of, - first_seen=now, - last_seen=now, - ) - captures[ch] = stored.model_dump() - self._register_content( - ch, - uh, - final_url, - blob_keys=( - None - if reuse - else { - "html": html_key, - "markdown": markdown_key, - "screenshot": screenshot_key, - } - ), - ) - - index["latest_content_hash"] = ch - index["last_checked"] = now - - # If the cited URL redirected to a different final URL, record the alias. - orig_uh = url_hash(result.url) - if orig_uh != uh: - aliases = index.setdefault("aliases", []) - if result.url not in aliases: - aliases.append(result.url) - - self._write_index(uh, index) - - if orig_uh != uh: - self._write_alias(orig_uh, result.url, uh, now) - - return StoreResult(capture=stored, created=created) - - def _write_alias( - self, orig_uh: str, orig_url: str, final_uh: str, now: str - ) -> None: - """Write/refresh a pointer from a cited URL's hash to its final capture. - - Never clobbers a canonical index (one that already holds captures), so a - URL fetched directly in the past keeps its own history. - """ - existing = self._read_index(orig_uh) - if existing and existing.get("captures"): - return - first_seen = existing.get("first_seen", now) if existing else now - self._write_index( - orig_uh, - { - "url": orig_url, - "url_hash": orig_uh, - "alias_of": final_uh, - "first_seen": first_seen, - "last_checked": now, - }, - ) diff --git a/forecasting_tools/agents_and_tools/source_archive/cost.py b/forecasting_tools/agents_and_tools/source_archive/cost.py deleted file mode 100644 index 33e950eb..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/cost.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Estimate what a capture run cost — per backend and per archived site. - -Self-hosted browsers (CloakBrowser / Playwright) and local PDF parsing are ~free; -the managed backends (Hyperbrowser, Firecrawl) bill per page. This turns a run's -outcomes into a cost breakdown so an operator can see what the paid backends are -costing per site archived. - -Costs are **estimates** from each vendor's public pricing applied to the -configured proxy mode — we record the backend that produced each capture, not the -live credit count — so treat them as close approximations, not billed amounts. -Only *successful* captures are priced; a paid backend call that then failed the -quality gate isn't attributed to a backend here (so this slightly under-counts). -""" - -from __future__ import annotations - -import json -from collections import Counter - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive import layout -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig - -# $ per vendor credit (public list pricing, 2026-06). -_FIRECRAWL_CREDIT_USD = 0.00083 -_HYPERBROWSER_CREDIT_USD = 0.001 - -# Backends that run on our own machine — no per-page charge. -_FREE_BACKENDS = {"cloakbrowser", "playwright", "pdf", ""} - - -def price_per_capture(fetcher: str, config: ArchiveConfig) -> float: - """Estimated $ for one successful capture by ``fetcher`` under ``config``.""" - f = (fetcher or "").lower() - if f in _FREE_BACKENDS: - return 0.0 - if f == "hyperbrowser": - credits = 10 if config.hyperbrowser_use_proxy else 1 - return credits * _HYPERBROWSER_CREDIT_USD - if f.startswith("firecrawl"): - basic = (config.firecrawl_proxy or "basic").lower() in ("", "basic") - return (1 if basic else 5) * _FIRECRAWL_CREDIT_USD - return 0.0 # unknown backend — assume free rather than invent a number - - -class BackendCost(BaseModel): - backend: str - captures: int - unit_usd: float - total_usd: float - - -class RunCost(BaseModel): - run_id: str | None = None - archived: int = 0 # sites we now hold (stored + deduped + cache_hit) - paid_captures: int = 0 # captures via a paid backend this run - total_usd: float = 0.0 - usd_per_archived: float = 0.0 - by_backend: list[BackendCost] = [] - - def __str__(self) -> str: - lines = [ - f"RunCost(run_id={self.run_id}, archived={self.archived}, " - f"paid_captures={self.paid_captures}, total=${self.total_usd:.4f}, " - f"$/archived=${self.usd_per_archived:.5f})", - f" {'backend':<14}{'captures':>9}{'$/capture':>12}{'$ total':>10}", - ] - for b in self.by_backend: - lines.append( - f" {b.backend:<14}{b.captures:>9}{b.unit_usd:>12.5f}{b.total_usd:>10.4f}" - ) - return "\n".join(lines) - - -def estimate_run_cost( - summary, config: ArchiveConfig, run_id: str | None = None -) -> RunCost: - """Estimate a :class:`PipelineSummary`'s cost, broken down by backend. - - Newly fetched captures (``stored`` / ``deduped``) are priced by the backend - that produced them; ``cache_hit`` re-uses cost nothing (no fetch happened). - """ - counts: Counter[str] = Counter() - for o in summary.outcomes: - if o.status in ("stored", "deduped") and o.stored is not None: - counts[(o.stored.fetcher or "unknown")] += 1 - - by_backend: list[BackendCost] = [] - total = 0.0 - paid = 0 - for backend, n in sorted(counts.items()): - unit = price_per_capture(backend, config) - sub = unit * n - total += sub - if unit > 0: - paid += n - by_backend.append( - BackendCost( - backend=backend, - captures=n, - unit_usd=round(unit, 6), - total_usd=round(sub, 4), - ) - ) - - archived = sum( - 1 for o in summary.outcomes if o.status in ("stored", "deduped", "cache_hit") - ) - return RunCost( - run_id=run_id, - archived=archived, - paid_captures=paid, - total_usd=round(total, 4), - usd_per_archived=round(total / archived, 6) if archived else 0.0, - by_backend=by_backend, - ) - - -def cost_report_key( - run_id: str, config: ArchiveConfig, group: str | None = None -) -> str: - prefix = config.s3_prefix.rstrip("/") - return f"{prefix}/{layout.report_key(run_id, '_cost.json', group)}" - - -def write_cost_report( - store, run_id: str, cost: RunCost, config: ArchiveConfig, group: str | None = None -) -> str: - """Persist the cost breakdown next to the run report (``_cost.json``).""" - key = cost_report_key(run_id, config, group) - store.put( - key, - json.dumps(cost.model_dump(), indent=2).encode("utf-8"), - content_type="application/json", - ) - return key diff --git a/forecasting_tools/agents_and_tools/source_archive/coverage.py b/forecasting_tools/agents_and_tools/source_archive/coverage.py deleted file mode 100644 index 9b812e9b..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/coverage.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Coverage reports: what fraction of cited sources did we actually archive? - -Two **separate** reports, split by ingestion path — they have different -denominators and different notions of ground truth, so they must not be blurred: - -- ``trace`` — the complex/template bot's own instrumented runs (metac-ai-sdk). - Traces record *every* URL the bot touched, so this is a true archival - success-rate against ground truth. -- ``comments`` — every bot (Metaculus's own + outside bots) harvested from public - Metaculus comments. Comments are length-truncated, so the denominator is itself - incomplete: coverage here means "of the links we could *see* in comments, how - many we archived" — a weaker claim than the trace report. - -For each mode: denominator = distinct canonical **page** sources cited (tool/API -calls excluded, same rule as the catalog); numerator = those with a successful -capture in the index. Misses are bucketed by site — the per-URL failure *reason* -isn't persisted yet (that needs each run's pipeline outcomes saved), so we can -say *which* sites we miss, not yet *why*. -""" - -from __future__ import annotations - -import csv -import io -from collections import defaultdict - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive.catalog import ( - Source, - build_sources, - exclusion_reason, -) -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) - -MODES = ("trace", "comments") -_COMMENT_ORIGINS = {"metaculus_comment"} - - -def citation_mode(citation) -> str: - return "comments" if (citation.origin or "") in _COMMENT_ORIGINS else "trace" - - -class CoverageRow(BaseModel): - label: str - cited: int = 0 - captured: int = 0 - - @property - def pct(self) -> float: - return round(100 * self.captured / self.cited, 1) if self.cited else 0.0 - - -class CoverageReport(BaseModel): - mode: str - cited: int = 0 - captured: int = 0 - excluded: dict[str, int] = {} # non-source reason -> count - by_question: list[CoverageRow] = [] - by_bot: list[CoverageRow] = [] - by_tool: list[CoverageRow] = [] - missed_by_domain: list[CoverageRow] = [] - missing_urls: list[str] = [] - # Populated only when per-run outcomes (reports/) exist: - has_outcomes: bool = False - missing_never_fetched: int = 0 # the real collection gap - missing_fetch_failed: int = 0 # tried, failed (Cloudflare/PDF/…) - - @property - def pct(self) -> float: - return round(100 * self.captured / self.cited, 1) if self.cited else 0.0 - - @property - def missing(self) -> int: - return self.cited - self.captured - - def __str__(self) -> str: - title = { - "trace": "Trace coverage — complex/template bot (ground truth)", - "comments": "Comment coverage — all bots (truncated denominator)", - }.get(self.mode, self.mode) - excl = ( - " (excluded as non-sources: " - + ", ".join(f"{k} {v}" for k, v in sorted(self.excluded.items())) - + ")" - if self.excluded - else "" - ) - lines = [ - title, - "=" * len(title), - # Lead with the collection gap: this report exists to tell us whether - # there are sources bots are using that we are NOT yet archiving. - f"{self.missing} of {self.cited} cited page sources are NOT yet in the " - f"archive — candidates to collect ({self.captured} archived, " - f"{self.pct}%).", - excl, - ] - if self.has_outcomes: - lines.append( - f" of those {self.missing}: {self.missing_never_fetched} were " - f"never fetched (collection gap), {self.missing_fetch_failed} " - f"were fetched but failed." - ) - if self.mode == "comments": - lines.append( - " note: comments are length-truncated, so even this denominator " - "under-counts what bots actually read — the true gap is larger." - ) - - def table(header: str, rows: list[CoverageRow], n: int = 8) -> None: - if not rows: - return - lines.append("") - lines.append(f"--- {header} ---") - for r in rows[:n]: - lines.append(f" {r.captured:>4}/{r.cited:<4} {r.pct:>5}% {r.label}") - if len(rows) > n: - lines.append(f" … +{len(rows) - n} more") - - table("by question (most-cited first)", self.by_question) - table("by bot", self.by_bot) - if self.mode == "trace": - table("by tool", self.by_tool) - table("biggest collection gaps by site (captured/cited)", self.missed_by_domain) - if self.missing_urls: - lines.append("") - lines.append( - f"--- {len(self.missing_urls)} source(s) to collect (first 10) ---" - ) - for u in self.missing_urls[:10]: - lines.append(f" {u}") - return "\n".join(lines) - - def to_csv(self) -> str: - buf = io.StringIO() - w = csv.writer(buf) - w.writerow(["group", "label", "cited", "captured", "pct"]) - w.writerow(["overall", self.mode, self.cited, self.captured, self.pct]) - for group, rows in ( - ("question", self.by_question), - ("bot", self.by_bot), - ("tool", self.by_tool), - ("missed_domain", self.missed_by_domain), - ): - for r in rows: - w.writerow([group, r.label, r.cited, r.captured, r.pct]) - return buf.getvalue() - - -def _grouped(scoped: list[tuple[Source, list]], key_of) -> list[CoverageRow]: - agg: dict[str, list[int]] = defaultdict(lambda: [0, 0]) - for source, cits in scoped: - keys = {k for k in (key_of(c) for c in cits) if k} or {"(none)"} - for k in keys: - agg[k][0] += 1 - if source.captured: - agg[k][1] += 1 - rows = [CoverageRow(label=k, cited=v[0], captured=v[1]) for k, v in agg.items()] - return sorted(rows, key=lambda r: (-r.cited, r.label)) - - -def coverage_from_sources( - sources: list[Source], mode: str, outcomes: dict[str, str] | None = None -) -> CoverageReport: - scoped: list[tuple[Source, list]] = [] - excluded: dict[str, int] = defaultdict(int) - for s in sources: - cits = [c for c in s.citations if citation_mode(c) == mode] - if not cits: - continue - reason = exclusion_reason(s.canonical_url, cits) - if reason: - excluded[reason] += 1 - continue - scoped.append((s, cits)) - - captured = sum(1 for s, _ in scoped if s.captured) - - never_fetched = failed = 0 - if outcomes is not None: - from forecasting_tools.agents_and_tools.source_archive.reports import ( - FAILED_STATUSES, - ) - - for s, _ in scoped: - if s.captured: - continue - status = outcomes.get(s.canonical_url) - if status is None: - never_fetched += 1 - elif status in FAILED_STATUSES: - failed += 1 - else: - failed += 1 - - domain_agg: dict[str, list[int]] = defaultdict(lambda: [0, 0]) - for s, _ in scoped: - domain_agg[s.domain][0] += 1 - if s.captured: - domain_agg[s.domain][1] += 1 - missed_by_domain = sorted( - ( - CoverageRow(label=d, cited=c, captured=cap) - for d, (c, cap) in domain_agg.items() - if cap < c - ), - key=lambda r: (-(r.cited - r.captured), r.label), - ) - - return CoverageReport( - mode=mode, - cited=len(scoped), - captured=captured, - excluded=dict(excluded), - by_question=_grouped(scoped, lambda c: c.question_id), - by_bot=_grouped(scoped, lambda c: c.bot), - by_tool=_grouped(scoped, lambda c: c.tool_name), - missed_by_domain=missed_by_domain, - missing_urls=sorted(s.canonical_url for s, _ in scoped if not s.captured), - has_outcomes=outcomes is not None, - missing_never_fetched=never_fetched, - missing_fetch_failed=failed, - ) - - -def build_coverage( - store: BlobStore, config: ArchiveConfig, mode: str -) -> CoverageReport: - from forecasting_tools.agents_and_tools.source_archive.reports import read_outcomes - - sources = build_sources(store, config) - outcomes = read_outcomes(store, config) or None - return coverage_from_sources(sources, mode, outcomes) diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py deleted file mode 100644 index e41b581d..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Fetchers turn a URL into a CaptureResult (HTML + screenshot + markdown). - -Most callers want :func:`build_default_fetcher`, which wires the recommended -cost-ordered tiered setup: self-hosted Playwright primary, then CloakBrowser, -PDF, Firecrawl, and Hyperbrowser backups. -""" - -from __future__ import annotations - -import logging - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import ( - Fetcher, - FetchError, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.cloakbrowser_fetcher import ( - CloakBrowserFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.firecrawl_fetcher import ( - FirecrawlFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.hyperbrowser_fetcher import ( - HyperbrowserFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.pdf_fetcher import ( - PdfFetcher, - looks_like_pdf, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.playwright_fetcher import ( - PlaywrightFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.fetchers.tiered import ( - TieredFetcher, -) - -logger = logging.getLogger(__name__) - -__all__ = [ - "Fetcher", - "FetchError", - "CloakBrowserFetcher", - "FirecrawlFetcher", - "HyperbrowserFetcher", - "PdfFetcher", - "PlaywrightFetcher", - "TieredFetcher", - "looks_like_pdf", - "build_default_fetcher", -] - - -def build_default_fetcher(config: ArchiveConfig | None = None) -> PlaywrightFetcher: - """Return the recommended fetcher as a context manager. - - Use it like:: - - with build_default_fetcher(config) as fetcher: - fetcher.fetch(url) - - Backends are tried in **cost order** — the first capture that passes the - quality gate wins, so the cheap tiers absorb most of the tail and the paid - ones only fire on what survives them: - - 1. **Self-hosted browser** (~free) — the primary; ~70% of URLs. Uses - **CloakBrowser** (patched Chromium; matches-or-beats vanilla Playwright on - anti-bot) when installed, else falls back to vanilla **Playwright**. Only - one browser is used: two live ``sync_playwright`` instances conflict in a - single process, so cloak *replaces* vanilla rather than stacking with it. - 2. **PdfFetcher** (local, free; Firecrawl OCR fallback) — captures PDFs, - which the browsers can't render. - 3. **Firecrawl** (managed) — cheapest stealth + native-PDF safety net - (~$0.0042/page stealth). Added when ``FIRECRAWL_API_KEY`` is set. - 4. **Hyperbrowser** (managed) — anti-bot fallback of last resort (~$0.01/page - with proxy, plus bandwidth). Added when ``HYPERBROWSER_API_KEY`` is set. - - The returned object is a :class:`PlaywrightFetcher` subclass so the single - browser's lifecycle is managed by ``with``. - """ - config = config or ArchiveConfig() - return _ManagedTieredFetcher(config) - - -class _ManagedTieredFetcher(PlaywrightFetcher): - """PlaywrightFetcher whose ``fetch`` is delegated to a cost-ordered tiered - pipeline. The single self-hosted browser is CloakBrowser when available - (overriding ``_launch_browser``), else vanilla Playwright; the extra backends - are composed once it is live. - """ - - _primary_name = "playwright" - - def _launch_browser(self): - # Prefer CloakBrowser (patched Chromium, beats vanilla on anti-bot) as - # the one self-hosted browser. Two live sync_playwright instances in a - # process conflict, so cloak REPLACES vanilla here rather than stacking. - try: - browser = CloakBrowserFetcher(self.config)._launch_browser() - self._primary_name = "cloakbrowser" - return browser - except FetchError as e: - logger.info("cloakbrowser unavailable, using vanilla Playwright: %s", e) - self._primary_name = "playwright" - return super()._launch_browser() - - def __enter__(self) -> "_ManagedTieredFetcher": - super().__enter__() # launches the chosen browser via _launch_browser - backends: list[Fetcher] = [_PrimaryBrowser(self, self._primary_name)] - - # PDFs: free local parse (Firecrawl OCR fallback wired internally when a - # key is present). Cheap to keep in the chain unconditionally. - backends.append(PdfFetcher(self.config)) - - if self.config.firecrawl_api_key: - backends.append(FirecrawlFetcher(self.config)) - if self.config.hyperbrowser_api_key: - backends.append(HyperbrowserFetcher(self.config)) - - self._tiered = TieredFetcher(*backends) - return self - - def fetch(self, url: str): # type: ignore[override] - return self._tiered.fetch(url) - - -class _PrimaryBrowser: - """Adapts the live primary browser to the Fetcher protocol for tiering, - calling the un-overridden ``fetch`` so we don't recurse, and labelling the - capture with the actual browser used (cloakbrowser/playwright).""" - - def __init__(self, owner: PlaywrightFetcher, name: str): - self._owner = owner - self.name = name - - def fetch(self, url: str): - result = PlaywrightFetcher.fetch(self._owner, url) - result.fetcher = self.name - return result diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/base.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/base.py deleted file mode 100644 index e2432a8a..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/base.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Fetcher interface. - -A fetcher turns a URL into a ``CaptureResult`` (HTML + markdown + screenshot in -one pass). Implementations: self-hosted Playwright (primary) and Firecrawl -(fallback). -""" - -from __future__ import annotations - -from typing import Protocol, runtime_checkable - -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - - -class FetchError(Exception): - """Raised when a fetcher cannot produce a capture at all (network/render - failure). Quality problems with an otherwise-successful fetch are not errors - — those are handled by the quality gate.""" - - -@runtime_checkable -class Fetcher(Protocol): - name: str - - def fetch(self, url: str) -> CaptureResult: ... diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/cloakbrowser_fetcher.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/cloakbrowser_fetcher.py deleted file mode 100644 index d4164e70..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/cloakbrowser_fetcher.py +++ /dev/null @@ -1,62 +0,0 @@ -"""CloakBrowser fetcher — a self-hosted anti-bot upgrade to Playwright. - -CloakBrowser (``CloakHQ/CloakBrowser``) is an open-source, patched-Chromium fork -whose ``cloakbrowser.launch()`` returns a standard Playwright ``Browser`` — so -this fetcher reuses *all* of ``PlaywrightFetcher``'s capture logic (settle, -autoscroll, full-page screenshot, trafilatura→markdown) and only overrides how -the browser is launched. The fork applies source-level fingerprint patches that -get past Cloudflare Turnstile and similar challenges that plain headless Chromium -trips; in the one rigorous 2026 anti-detect benchmark it cleared more Cloudflare -targets than vanilla Playwright. - -It runs on your own compute, so the marginal service cost is ~$0/page. The -patched Chromium binary (~200MB) is downloaded automatically on first launch. - -Install separately (it is not in the ``source-archive`` extra because of the -binary): ``pip install cloakbrowser``. The package module is configurable via -``config.cloakbrowser_import`` (default ``cloakbrowser``) in case it is renamed. -""" - -from __future__ import annotations - -import importlib -import logging - -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.fetchers.playwright_fetcher import ( - PlaywrightFetcher, -) - -logger = logging.getLogger(__name__) - - -class CloakBrowserFetcher(PlaywrightFetcher): - name = "cloakbrowser" - - def _launch_browser(self): - module = self._import_module() - launch = getattr(module, "launch", None) - if launch is None: - raise FetchError( - f"{module.__name__} has no launch(); the CloakBrowser API may " - "have changed. Expected `cloakbrowser.launch() -> Browser`." - ) - # stealth_args=True applies the fingerprint patches; the returned object - # is a Playwright Browser, so the inherited fetch() drives it unchanged. - # No separate playwright handle to stop — CloakBrowser owns its driver. - browser = launch(headless=True, stealth_args=True) - return None, browser - - def _import_module(self): - candidates = [self.config.cloakbrowser_import, "cloakbrowser"] - tried: list[str] = [] - for mod_name in dict.fromkeys(c for c in candidates if c): - try: - return importlib.import_module(mod_name) - except ImportError: - tried.append(mod_name) - raise FetchError( - "cloakbrowser is not installed. Install it with " - "`pip install cloakbrowser` (or set WEB_ARCHIVE_CLOAKBROWSER_IMPORT " - f"to the right module). Tried: {', '.join(tried)}." - ) diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/firecrawl_fetcher.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/firecrawl_fetcher.py deleted file mode 100644 index 622d51ff..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/firecrawl_fetcher.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Firecrawl fetcher — a managed FALLBACK backend. - -Reserved for sites that block headless Chromium. A basic scrape costs 1 credit/ -page even with a screenshot, so it only runs when the primary backend fails or -its capture fails the quality gate. - -For hardened anti-bot sites, set ``config.firecrawl_proxy`` to ``"auto"`` or -``"stealth"`` (a.k.a. "enhanced") — this routes through residential proxies and -is billed at ~5 credits/page, so it is opt-in and reserved for the Cloudflare -tier. Firecrawl also natively parses PDFs to markdown (1 credit per PDF page), -which is why it is the fallback for the tiered ``PdfFetcher``. - -The Firecrawl SDK is optional and imported lazily. The screenshot comes back as -a hosted URL, which we download to bytes. -""" - -from __future__ import annotations - -import logging -import urllib.request - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - -logger = logging.getLogger(__name__) - - -def _attr(obj, key, default=None): - if obj is None: - return default - if isinstance(obj, dict): - return obj.get(key, default) - return getattr(obj, key, default) - - -class FirecrawlFetcher: - name = "firecrawl" - - def __init__(self, config: ArchiveConfig | None = None, client=None): - self.config = config or ArchiveConfig() - self._client = client - - def _get_client(self): - if self._client is not None: - return self._client - if not self.config.firecrawl_api_key: - raise FetchError("FIRECRAWL_API_KEY is not set") - try: - from firecrawl import Firecrawl - except ImportError as e: - raise FetchError( - "firecrawl-py is not installed. Install it with " - "`pip install forecasting-tools[source-archive]`." - ) from e - self._client = Firecrawl(api_key=self.config.firecrawl_api_key) - return self._client - - def _scrape_kwargs(self, formats: list[str]) -> dict: - kwargs: dict = {"formats": formats} - # Firecrawl 4.x renamed "stealth" to the "enhanced" proxy mode but still - # accepts the legacy spelling; pass whatever the operator configured and - # let the SDK normalize. "basic" is the implicit default, so only send - # the param when something stronger is requested (keeps the call 1-credit - # unless the operator explicitly opts into the 5-credit proxy). - proxy = (self.config.firecrawl_proxy or "basic").strip().lower() - if proxy and proxy != "basic": - kwargs["proxy"] = proxy - return kwargs - - def fetch(self, url: str) -> CaptureResult: - client = self._get_client() - try: - doc = client.scrape( - url, **self._scrape_kwargs(["markdown", "html", "screenshot"]) - ) - except Exception as e: - raise FetchError(f"firecrawl scrape failed for {url}: {e}") from e - - metadata = _attr(doc, "metadata", {}) or {} - status = _attr(metadata, "statusCode") or _attr(metadata, "status_code") - final_url = _attr(metadata, "sourceURL") or _attr(metadata, "url") or url - - screenshot_url = _attr(doc, "screenshot") - screenshot, content_type = None, None - if screenshot_url: - screenshot, content_type = self._download(screenshot_url) - - return CaptureResult( - url=url, - final_url=final_url, - status_code=int(status) if status is not None else None, - html=_attr(doc, "html"), - markdown=_attr(doc, "markdown"), - screenshot=screenshot, - screenshot_content_type=content_type, - fetcher=self.name, - metadata={ - "title": _attr(metadata, "title"), - "firecrawl_proxy": self.config.firecrawl_proxy, - }, - ) - - def fetch_pdf_markdown(self, url: str) -> str | None: - """Scrape just the markdown for a PDF URL via Firecrawl's native PDF - parser. Used as the fallback inside :class:`PdfFetcher` when local - extraction yields thin text (e.g. a scanned PDF needing OCR).""" - client = self._get_client() - try: - doc = client.scrape(url, **self._scrape_kwargs(["markdown"])) - except Exception as e: - raise FetchError(f"firecrawl pdf scrape failed for {url}: {e}") from e - return _attr(doc, "markdown") - - @staticmethod - def _download(src_url: str) -> tuple[bytes | None, str | None]: - try: - with urllib.request.urlopen(src_url, timeout=30) as resp: - return resp.read(), resp.headers.get("Content-Type", "image/png") - except Exception as e: - logger.warning("failed to download firecrawl screenshot: %s", e) - return None, None diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/hyperbrowser_fetcher.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/hyperbrowser_fetcher.py deleted file mode 100644 index ce728abd..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/hyperbrowser_fetcher.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Hyperbrowser fetcher — a managed FALLBACK backend. - -Hyperbrowser exposes a Firecrawl-style ``scrape`` endpoint that returns -markdown + HTML + a screenshot in one call, with optional stealth, residential -proxy, and CAPTCHA-solving session options for getting past Cloudflare and other -anti-bot filters. - -Why it's here even though Firecrawl already is: forecasting-tools already uses -Hyperbrowser elsewhere (``research/computer_use.py``), so routing the anti-bot -tail through it consolidates spend onto one vendor/bill. - -Cost note: a basic scrape is 1 credit ($0.001); enabling ``use_proxy`` makes it -10 credits ($0.01) plus proxy bandwidth ($10/GB). So the proxy/stealth session -is opt-in and meant for the small hardened-Cloudflare residual, not every URL. -Hyperbrowser has no documented PDF→markdown path, so PDFs go to the dedicated -``PdfFetcher`` instead of here. - -The SDK is optional and imported lazily; a screenshot may come back as a hosted -URL (downloaded to bytes) or inline base64. -""" - -from __future__ import annotations - -import base64 -import binascii -import logging -import urllib.request - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - -logger = logging.getLogger(__name__) - - -def _attr(obj, key, default=None): - if obj is None: - return default - if isinstance(obj, dict): - return obj.get(key, default) - return getattr(obj, key, default) - - -class HyperbrowserFetcher: - name = "hyperbrowser" - - def __init__(self, config: ArchiveConfig | None = None, client=None): - self.config = config or ArchiveConfig() - self._client = client - - def _get_client(self): - if self._client is not None: - return self._client - if not self.config.hyperbrowser_api_key: - raise FetchError("HYPERBROWSER_API_KEY is not set") - try: - from hyperbrowser import Hyperbrowser - except ImportError as e: - raise FetchError( - "hyperbrowser is not installed. Install it with " - "`pip install forecasting-tools[source-archive]`." - ) from e - self._client = Hyperbrowser(api_key=self.config.hyperbrowser_api_key) - return self._client - - def _params(self, url: str): - """Build the SDK request objects. Imported here (not at module top) so - importing this module never requires the SDK.""" - from hyperbrowser.models import ( - CreateSessionParams, - ScrapeOptions, - StartScrapeJobParams, - ) - - return StartScrapeJobParams( - url=url, - scrape_options=ScrapeOptions( - formats=["markdown", "html", "screenshot"], - only_main_content=False, - ), - session_options=CreateSessionParams( - use_proxy=self.config.hyperbrowser_use_proxy, - use_stealth=self.config.hyperbrowser_use_stealth, - solve_captchas=self.config.hyperbrowser_solve_captchas, - ), - ) - - def fetch(self, url: str) -> CaptureResult: - client = self._get_client() - try: - resp = client.scrape.start_and_wait(self._params(url)) - except Exception as e: - raise FetchError(f"hyperbrowser scrape failed for {url}: {e}") from e - - # The job wrapper carries status/error; the payload is on ``.data``. - if _attr(resp, "status") == "failed": - raise FetchError( - f"hyperbrowser scrape failed for {url}: {_attr(resp, 'error')}" - ) - data = _attr(resp, "data", resp) - - metadata = _attr(data, "metadata", {}) or {} - status = _attr(metadata, "statusCode") or _attr(metadata, "status_code") - final_url = _attr(metadata, "sourceURL") or _attr(metadata, "url") or url - - screenshot, content_type = self._coerce_screenshot(_attr(data, "screenshot")) - - return CaptureResult( - url=url, - final_url=final_url, - status_code=int(status) if status is not None else None, - html=_attr(data, "html"), - markdown=_attr(data, "markdown"), - screenshot=screenshot, - screenshot_content_type=content_type, - fetcher=self.name, - metadata={ - "title": _attr(metadata, "title"), - "used_proxy": self.config.hyperbrowser_use_proxy, - }, - ) - - @classmethod - def _coerce_screenshot(cls, value) -> tuple[bytes | None, str | None]: - """A screenshot may arrive as a hosted URL, a data: URI, or raw base64.""" - if not value or not isinstance(value, str): - return None, None - if value.startswith("http://") or value.startswith("https://"): - return cls._download(value) - if value.startswith("data:"): - try: - header, b64 = value.split(",", 1) - ctype = header[5:].split(";", 1)[0] or "image/png" - return base64.b64decode(b64), ctype - except (ValueError, binascii.Error): - return None, None - try: - return base64.b64decode(value, validate=True), "image/png" - except (binascii.Error, ValueError): - return None, None - - @staticmethod - def _download(src_url: str) -> tuple[bytes | None, str | None]: - try: - with urllib.request.urlopen(src_url, timeout=30) as resp: - return resp.read(), resp.headers.get("Content-Type", "image/png") - except Exception as e: - logger.warning("failed to download hyperbrowser screenshot: %s", e) - return None, None diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/pdf_fetcher.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/pdf_fetcher.py deleted file mode 100644 index 0977605c..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/pdf_fetcher.py +++ /dev/null @@ -1,146 +0,0 @@ -"""PDF fetcher — closes the gap Playwright can't. - -Headless Chromium *downloads* a PDF instead of rendering it (``page.goto`` raises -"Download is starting"), and trafilatura doesn't parse PDFs, so a cited ``.pdf`` -URL produces nothing today. This fetcher fills that hole with a two-tier strategy: - - 1. Download the PDF bytes and parse locally with **PyMuPDF4LLM** — free, fast, - CPU-only, and excellent on text-layer PDFs (most government/legal reports). - The first page is rendered to an image so the viewer still has a screenshot. - 2. If local extraction yields thin text (a scanned PDF that needs OCR), fall - back to **Firecrawl's** native PDF parser (~1 credit/page, OCR included). - -Both parsers are optional and imported lazily. Use :func:`looks_like_pdf` / -:meth:`PdfFetcher.is_pdf` to decide whether a URL should be routed here. -""" - -from __future__ import annotations - -import logging -import urllib.request - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.fetchers.firecrawl_fetcher import ( - FirecrawlFetcher, -) -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult -from forecasting_tools.agents_and_tools.source_archive.quality import MIN_TEXT_LEN - -logger = logging.getLogger(__name__) - -_PDF_MAGIC = b"%PDF-" - - -def looks_like_pdf(url: str) -> bool: - """Cheap URL-shape heuristic: does this look like a PDF link? (The fetcher - still confirms by sniffing the magic bytes before parsing.)""" - path = url.split("?", 1)[0].split("#", 1)[0].lower() - return path.endswith(".pdf") - - -class PdfFetcher: - name = "pdf" - - def __init__( - self, - config: ArchiveConfig | None = None, - *, - firecrawl: FirecrawlFetcher | None = None, - downloader=None, - ): - self.config = config or ArchiveConfig() - # Reuse the configured Firecrawl client for the OCR fallback when a key - # is present; otherwise the fallback is simply skipped. - if firecrawl is not None: - self._firecrawl = firecrawl - elif self.config.firecrawl_api_key: - self._firecrawl = FirecrawlFetcher(self.config) - else: - self._firecrawl = None - self._download = downloader or _download_bytes - - def is_pdf(self, url: str, data: bytes | None = None) -> bool: - if data is not None: - return data[:5] == _PDF_MAGIC - return looks_like_pdf(url) - - def fetch(self, url: str) -> CaptureResult: - data, final_url, status = self._download(url, self.config.nav_timeout_ms) - if not data or data[:5] != _PDF_MAGIC: - raise FetchError(f"not a PDF (no %PDF- magic) for {url}") - - markdown, screenshot, ctype, pages, engine = self._parse_local(data) - - thin = not markdown or len(markdown.strip()) < MIN_TEXT_LEN - if thin and self._firecrawl is not None: - logger.info("local PDF parse thin for %s; trying Firecrawl OCR", url) - try: - fc_md = self._firecrawl.fetch_pdf_markdown(url) - except FetchError as e: - logger.info("firecrawl PDF fallback failed for %s: %s", url, e) - else: - if fc_md and len(fc_md.strip()) >= MIN_TEXT_LEN: - markdown, engine = fc_md, "firecrawl" - - return CaptureResult( - url=url, - final_url=final_url or url, - status_code=status, - html=None, - markdown=markdown, - screenshot=screenshot, - screenshot_content_type=ctype, - fetcher=self.name, - metadata={"pdf_engine": engine, "pdf_pages": pages}, - ) - - def _parse_local( - self, data: bytes - ) -> tuple[str | None, bytes | None, str | None, int, str]: - """Return (markdown, screenshot_png, content_type, pages, engine).""" - try: - import pymupdf # PyMuPDF (a.k.a. fitz) - import pymupdf4llm - except ImportError: - logger.warning( - "pymupdf4llm not installed; local PDF parsing unavailable. " - "Install with `pip install forecasting-tools[source-archive]`." - ) - return None, None, None, 0, "none" - - try: - doc = pymupdf.open(stream=data, filetype="pdf") - except Exception as e: - raise FetchError(f"could not open PDF: {e}") from e - - try: - total = doc.page_count - limit = min(total, self.config.pdf_max_pages) or total - markdown = pymupdf4llm.to_markdown(doc, pages=list(range(limit))) - screenshot, ctype = self._render_first_page(doc) - return markdown or None, screenshot, ctype, total, "pymupdf4llm" - finally: - doc.close() - - @staticmethod - def _render_first_page(doc) -> tuple[bytes | None, str | None]: - try: - pix = doc[0].get_pixmap(dpi=110) - return pix.tobytes("png"), "image/png" - except Exception as e: - logger.info("could not render PDF first page: %s", e) - return None, None - - -def _download_bytes( - url: str, timeout_ms: int -) -> tuple[bytes | None, str | None, int | None]: - # A browser-ish UA avoids the cheapest 403s; the content store needs the - # bytes, not a render, so plain HTTP is fine and free. - req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - try: - with urllib.request.urlopen(req, timeout=max(timeout_ms / 1000, 1)) as resp: - return resp.read(), resp.geturl(), getattr(resp, "status", 200) - except Exception as e: - raise FetchError(f"could not download PDF for {url}: {e}") from e diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/playwright_fetcher.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/playwright_fetcher.py deleted file mode 100644 index d81ea82e..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/playwright_fetcher.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Self-hosted Playwright fetcher — the PRIMARY backend. - -A single page load yields all three artifacts: - - - HTML via ``page.content()`` - - screenshot via a full-page capture (height-capped, then compressed) - - markdown via trafilatura over the rendered HTML - -Self-hosted compute is far cheaper than any managed scraping API, so this is the -default; Firecrawl is reserved for sites that block headless Chromium (see -``TieredFetcher``). - -Playwright and trafilatura are optional and imported lazily, so importing this -module never requires a browser. Install everything with -``pip install forecasting-tools[source-archive]`` and then run -``playwright install chromium`` once to download the browser. -""" - -from __future__ import annotations - -import io -import logging - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import FetchError -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - -logger = logging.getLogger(__name__) - -# WebP's hard per-side pixel limit; taller captures must be cropped before encode. -_WEBP_MAX_DIM = 16383 -# Above this total pixel count, skip the screenshot rather than decode it: a -# pathological full-page render (very tall × wide) costs minutes of CPU in Pillow -# for a screenshot that's nice-to-have, not essential. -_MAX_SCREENSHOT_PIXELS = 200_000_000 - - -def _to_markdown(html: str, url: str) -> str | None: - try: - import trafilatura - except ImportError: - logger.warning("trafilatura not installed; markdown will be omitted") - return None - return trafilatura.extract( - html, url=url, output_format="markdown", include_links=True - ) - - -# Scroll the document top-to-bottom (triggering lazy-loaded content) then back -# up, so a subsequent full-page screenshot captures the fully-rendered page. -_AUTOSCROLL_JS = """ -async () => { - await new Promise((resolve) => { - let y = 0; - const step = () => { - window.scrollTo(0, y); - y += 1000; - if (y < document.body.scrollHeight) setTimeout(step, 40); - else resolve(); - }; - step(); - }); - window.scrollTo(0, 0); -} -""" - - -def _encode_screenshot( - png_bytes: bytes, fmt: str, max_height: int = 0 -) -> tuple[bytes, str]: - """Crop (to ``max_height``) and re-encode a PNG screenshot using Pillow. - - Pillow is already a forecasting-tools dependency, so true WebP is available - here (Playwright itself only emits PNG/JPEG). The height cap is enforced by - cropping the *full-page* render to its top ``max_height`` pixels — never via - Playwright's ``clip`` (which, without ``full_page``, is bounded by the - viewport and silently truncates tall pages to a single screen). - """ - fmt = fmt.lower() - try: - from PIL import Image - except ImportError: - # No Pillow: can't crop or transcode; hand back the raw full-page PNG. - return png_bytes, "image/png" - - image = Image.open(io.BytesIO(png_bytes)) # lazy: reads size, doesn't decode - if image.width * image.height > _MAX_SCREENSHOT_PIXELS: - raise ValueError( - f"screenshot too large to encode ({image.width}x{image.height}px)" - ) - # WebP cannot encode beyond 16383px on a side. Clamp the effective cap for - # webp so an over-tall page degrades to a top-crop instead of crashing the - # encoder mid-run (which would propagate out of fetch() and abort the URL). - limit = max_height or 0 - if fmt == "webp": - limit = min(limit or _WEBP_MAX_DIM, _WEBP_MAX_DIM) - if limit and image.height > limit: - image = image.crop((0, 0, image.width, limit)) - - out = io.BytesIO() - if fmt == "webp": - image.save(out, format="WEBP", quality=80, method=6) - return out.getvalue(), "image/webp" - if fmt in ("jpeg", "jpg"): - image.convert("RGB").save(out, format="JPEG", quality=80, optimize=True) - return out.getvalue(), "image/jpeg" - image.save(out, format="PNG", optimize=True) - return out.getvalue(), "image/png" - - -class PlaywrightFetcher: - """Renders pages with a persistent headless Chromium. - - Use it as a context manager so the browser launches once and is reused - across many URLs (throughput is thousands of pages/hour single-process):: - - with PlaywrightFetcher(config) as fetcher: - for url in urls: - fetcher.fetch(url) - """ - - name = "playwright" - - def __init__(self, config: ArchiveConfig | None = None): - self.config = config or ArchiveConfig() - self._playwright = None - self._browser = None - - def _launch_browser(self): - """Start the browser. Returns ``(playwright_or_none, browser)`` where - ``browser`` is a Playwright ``Browser``. Subclasses override this to swap - in a different stealth browser (see ``CloakBrowserFetcher``) while reusing - all of the capture logic. A backend that manages its own driver returns - ``None`` for the first element.""" - try: - from playwright.sync_api import sync_playwright - except ImportError as e: - raise FetchError( - "playwright is not installed. Install it with " - "`pip install forecasting-tools[source-archive]` and then run " - "`playwright install chromium`." - ) from e - playwright = sync_playwright().start() - browser = playwright.chromium.launch(headless=True) - return playwright, browser - - def __enter__(self) -> "PlaywrightFetcher": - self._playwright, self._browser = self._launch_browser() - return self - - def __exit__(self, *exc) -> None: - # close() raises when the browser process is already gone (crashed or - # killed by the pipeline's reaper). Still attempt stop(): it tears down - # the driver and un-registers the sync API's event loop from this - # thread — without that, the next sync_playwright().start() here fails - # with "Sync API inside the asyncio loop". - try: - if self._browser is not None: - self._browser.close() - except Exception as e: - logger.info("browser close failed (already dead?): %s", e) - finally: - self._browser = None - try: - if self._playwright is not None: - self._playwright.stop() - except Exception as e: - logger.info("playwright stop failed: %s", e) - finally: - self._playwright = None - - def _settle(self, page) -> None: - """Best-effort: let the page finish rendering before the screenshot. - - ``page.goto`` only waits for ``domcontentloaded``, which fires before - CSS/images/lazy content have laid out — capturing then yields a short, - half-built page. Wait for the load/network to quiesce and scroll the - document to force lazy content in, so the full-page capture is complete. - Each step is bounded and swallows timeouts: rendering aids are - nice-to-have, never fatal to the capture. - """ - try: - page.wait_for_load_state("load", timeout=self.config.nav_timeout_ms) - except Exception: - pass - try: - page.wait_for_load_state( - "networkidle", timeout=min(self.config.nav_timeout_ms, 10_000) - ) - except Exception: - pass - try: - page.evaluate(_AUTOSCROLL_JS) - page.wait_for_timeout(500) - except Exception: - pass - - def fetch(self, url: str) -> CaptureResult: - if self._browser is None: - raise FetchError("PlaywrightFetcher must be used as a context manager") - - context = self._browser.new_context() - page = context.new_page() - try: - try: - response = page.goto( - url, - wait_until="domcontentloaded", - timeout=self.config.nav_timeout_ms, - ) - except Exception as e: - raise FetchError(f"navigation failed for {url}: {e}") from e - - self._settle(page) - - status = response.status if response is not None else None - html = page.content() - - # Always capture the entire scrollable page in one shot — Playwright - # stitches it internally. The height cap is applied afterward by - # cropping in Pillow (see ``_encode_screenshot``). Fall back to a - # viewport capture only if a full-page shot fails (e.g. a page taller - # than Chromium's screenshot limit). - try: - png = page.screenshot(full_page=True) - except Exception as e: - logger.info("full-page screenshot failed for %s: %s", url, e) - png = page.screenshot() - # Encoding can fail on pathological pages (e.g. a 400M-pixel full-page - # render trips Pillow's decompression-bomb guard). A screenshot is - # nice-to-have — never lose the whole capture over it. - try: - screenshot, content_type = _encode_screenshot( - png, - self.config.screenshot_format, - self.config.screenshot_max_height, - ) - except Exception as e: - logger.info("screenshot encode failed for %s: %s", url, e) - screenshot, content_type = None, None - - return CaptureResult( - url=url, - final_url=page.url, - status_code=status, - html=html, - markdown=_to_markdown(html, page.url), - screenshot=screenshot, - screenshot_content_type=content_type, - fetcher=self.name, - metadata={"title": page.title()}, - ) - finally: - context.close() diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/tiered.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/tiered.py deleted file mode 100644 index bb47640a..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/tiered.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Tiered fetcher: self-hosted Playwright first, Firecrawl on failure. - -A backend "fails" if it raises ``FetchError`` (couldn't render) OR its capture -fails the quality gate (404 / block page / thin content). The first capture that -passes the gate wins. If none pass, the last attempted capture is returned with -``quality_passed=False`` in its metadata so the pipeline can still record the -miss. -""" - -from __future__ import annotations - -import logging - -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import ( - Fetcher, - FetchError, -) -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult -from forecasting_tools.agents_and_tools.source_archive.quality import evaluate - -logger = logging.getLogger(__name__) - - -class TieredFetcher: - name = "tiered" - - def __init__(self, *backends: Fetcher): - if not backends: - raise ValueError("TieredFetcher requires at least one backend") - self.backends = backends - - def fetch(self, url: str) -> CaptureResult: - last_result: CaptureResult | None = None - errors: list[str] = [] - - for backend in self.backends: - try: - result = backend.fetch(url) - except FetchError as e: - errors.append(f"{backend.name}: {e}") - continue - - verdict = evaluate(result) - result.metadata["quality_passed"] = verdict.passed - result.metadata["quality_reason"] = verdict.reason - if verdict.passed: - return result - last_result = result - errors.append(f"{backend.name}: quality {verdict.reason}") - - if last_result is not None: - logger.info( - "all backends failed quality for %s: %s", url, "; ".join(errors) - ) - return last_result - raise FetchError(f"all backends failed for {url}: {'; '.join(errors)}") diff --git a/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py b/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py deleted file mode 100644 index 5b9729db..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Ingestion: discover the URLs a bot cited and turn them into a manifest. - -The capture pipeline needs a citation manifest as input. These helpers build one -from a bot's published reasoning: - - - :mod:`url_extraction` — pull URLs out of free text / markdown. - - :mod:`trace_extraction` — build a manifest from a traced bot run (fullest path). -""" - -from forecasting_tools.agents_and_tools.source_archive.ingest.trace_extraction import ( - extract_records_from_events, - extract_records_from_question_dir, - extract_records_from_trace_file, - harvest_run, -) -from forecasting_tools.agents_and_tools.source_archive.ingest.url_extraction import ( - dedupe_records, - extract_citation_records, - extract_urls, -) - -__all__ = [ - "dedupe_records", - "extract_citation_records", - "extract_records_from_events", - "extract_records_from_question_dir", - "extract_records_from_trace_file", - "extract_urls", - "harvest_run", -] diff --git a/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py b/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py deleted file mode 100644 index 1c63fe08..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py +++ /dev/null @@ -1,379 +0,0 @@ -"""Build a citation manifest from a bot's run traces. - -When the template bot is run with tracing enabled it writes one JSONL trace per -forecast attempt, recording the agent loop step by step. Those traces are the -*fullest* record of what the bot actually looked at — richer than the reasoning -comment it posts, which is length-truncated. - -This module walks those traces and pulls out every external URL the bot touched, -turning each into a :class:`CitationRecord` with provenance (which trace, which -tool, the search query that surfaced it). That manifest is the input to the -capture pipeline, exactly like the comment-harvested one. - -Trace layout ------------- -A traced run is a directory tree:: - - / - bot_/ - q_/ - question.json - traces_forecast_1_attempt_1.jsonl - traces_summarize.jsonl - ... - -Each ``traces_*.jsonl`` file is a stream of newline-delimited event objects. The -events that can carry external links are: - -- ``tool_call`` — the arguments the bot passed to a tool (e.g. a search query, - or a ``url`` handed to a page fetcher). Carries ``name`` and ``call_id``. -- ``tool_result`` — what the tool returned. Search tools inline their citations - here as ``[n](url)`` or as a list of result URLs. Carries ``call_id`` so the - result can be attributed back to the originating ``tool_call``. -- ``initial_prompt`` — the first prompt of a trace. Only scanned for the - ``summarize`` trace: the template bot runs research *outside* the agent loop - and pastes the research blob verbatim into the summarizer's first prompt, so - that is the one place those URLs are recoverable. Other traces' initial - prompts just echo the question text (background, resolution criteria), whose - URLs aren't research, so they're skipped. - -Search provenance (``query`` / ``tool_args``) only exists in these instrumented -traces — it is populated here from each ``tool_call`` and carried onto the URLs -that the matching ``tool_result`` returned. -""" - -from __future__ import annotations - -import glob -import json -import os -from pathlib import Path -from typing import Any - -from forecasting_tools.agents_and_tools.source_archive.ingest.url_extraction import ( - extract_urls, -) -from forecasting_tools.agents_and_tools.source_archive.models import CitationRecord - -METACULUS_QUESTION_URL_FMT = "https://www.metaculus.com/questions/{}/" - -# Event type -> the field on that event that carries the URL-bearing payload. -_SCANNABLE_FIELDS: dict[str, str] = { - "tool_call": "args", - "tool_result": "content", - "initial_prompt": "prompt", -} -# The trace whose initial prompt holds pasted-in research (see module docstring). -_SUMMARIZE_TRACE_LABEL = "summarize" -# Keys a tool's input commonly uses for the search string, best-effort. -_QUERY_KEYS = ("query", "q", "search_query", "search", "queries", "question") - - -def _urls_in(value: Any) -> list[str]: - """Return URLs found anywhere in a string / dict / list, in first-seen order. - - Tool args are structured (a dict) and tool results may be either a blob of - text or a structured payload, so we walk the whole value and run the shared - :func:`extract_urls` over every string we reach — keeping markdown-link and - trailing-punctuation handling identical to the comment path. - """ - urls: list[str] = [] - - def walk(v: Any) -> None: - if v is None: - return - if isinstance(v, str): - urls.extend(extract_urls(v)) - return - if isinstance(v, dict): - for key, val in v.items(): - walk(key) - walk(val) - return - if isinstance(v, (list, tuple, set, frozenset)): - for item in v: - walk(item) - return - walk(str(v)) - - walk(value) - return urls - - -def _query_from_args(args: Any) -> str | None: - """Pull the search string out of a tool's arguments, if recognisable.""" - if not isinstance(args, dict): - return None - for key in _QUERY_KEYS: - val = args.get(key) - if isinstance(val, str) and val.strip(): - return val - if isinstance(val, (list, tuple)) and val: - joined = " ".join(str(item) for item in val if item) - if joined.strip(): - return joined - return None - - -def trace_label(trace_path: str) -> str: - """``traces_forecast_1_attempt_1.jsonl`` -> ``forecast_1_attempt_1``.""" - name = os.path.basename(trace_path) - if name.startswith("traces_"): - name = name[len("traces_") :] - if name.endswith(".jsonl"): - name = name[: -len(".jsonl")] - return name - - -def extract_records_from_events( - events: Any, - *, - trace: str | None = None, - include_initial_prompt: bool = False, - run_id: str | None = None, - bot: str | None = None, - question_id: str | None = None, - metaculus_id: str | None = None, - question_url: str | None = None, -) -> list[CitationRecord]: - """Turn one trace's event stream into CitationRecords. - - ``events`` is any iterable of event dicts (already parsed from JSONL). The - given provenance is stamped onto every record; per-event provenance - (``trace``, ``tool_name``, ``origin``, ``query``, ``tool_args``, - ``first_seen``) is filled in here. - - Set ``include_initial_prompt`` to scan ``initial_prompt`` events — callers - should only do this for the ``summarize`` trace (see module docstring). - """ - records: list[CitationRecord] = [] - # Attribute tool_result events (which only carry call_id) back to the - # originating tool_call's name and arguments. - tool_name_by_call_id: dict[str, str] = {} - tool_args_by_call_id: dict[str, Any] = {} - - for event in events: - if not isinstance(event, dict): - continue - event_type = event.get("type") - - if event_type == "tool_call": - call_id = str(event.get("call_id") or "").strip() - name = event.get("name") or "" - if call_id: - if name: - tool_name_by_call_id[call_id] = name - if "args" in event: - tool_args_by_call_id[call_id] = event.get("args") - - field = _SCANNABLE_FIELDS.get(event_type or "") - if field is None: - continue - if event_type == "initial_prompt" and not include_initial_prompt: - continue - - urls = _urls_in(event.get(field)) - if not urls: - continue - - if event_type == "tool_call": - tool_name = event.get("name") or "" - origin = "tool_call" - tool_args = ( - event.get("args") if isinstance(event.get("args"), dict) else None - ) - elif event_type == "tool_result": - call_id = str(event.get("call_id") or "").strip() - tool_name = tool_name_by_call_id.get(call_id, "") - origin = "tool_result" - originating_args = tool_args_by_call_id.get(call_id) - tool_args = originating_args if isinstance(originating_args, dict) else None - else: # initial_prompt - tool_name = "" - origin = event_type or "" - tool_args = None - - query = _query_from_args(tool_args) - timestamp = event.get("timestamp") - for url in urls: - record = CitationRecord( - url=url, - run_id=run_id, - bot=bot, - question_id=question_id, - metaculus_id=metaculus_id, - question_url=question_url, - trace=trace, - tool_name=tool_name, - origin=origin, - query=query, - tool_args=tool_args, - ) - if timestamp: - record.first_seen = str(timestamp) - records.append(record) - - return records - - -def _read_jsonl(path: str) -> list[dict]: - """Read a JSONL file, skipping blank or unparsable lines.""" - events: list[dict] = [] - for raw_line in Path(path).read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line: - continue - try: - events.append(json.loads(line)) - except json.JSONDecodeError: - continue - return events - - -def extract_records_from_trace_file( - trace_path: str, - *, - run_id: str | None = None, - bot: str | None = None, - question_id: str | None = None, - metaculus_id: str | None = None, - question_url: str | None = None, -) -> list[CitationRecord]: - """Extract CitationRecords from one ``traces_*.jsonl`` file.""" - label = trace_label(trace_path) - return extract_records_from_events( - _read_jsonl(trace_path), - trace=label, - include_initial_prompt=(label == _SUMMARIZE_TRACE_LABEL), - run_id=run_id, - bot=bot, - question_id=question_id, - metaculus_id=metaculus_id, - question_url=question_url, - ) - - -def _read_question_metadata(question_dir: str) -> tuple[str | None, str | None]: - """Return ``(question_id, metaculus_id)`` from ``question.json`` in the dir. - - Read as a plain dict with flexible keys so the ingest stays decoupled from - any particular question model. Missing/unparsable metadata is non-fatal — - records are still emitted, just with empty question provenance. - """ - question_path = os.path.join(question_dir, "question.json") - if not os.path.exists(question_path): - return None, None - try: - data = json.loads(Path(question_path).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return None, None - if not isinstance(data, dict): - return None, None - - def _str_or_none(*keys: str) -> str | None: - for key in keys: - val = data.get(key) - if val is not None: - return str(val) - return None - - question_id = _str_or_none("question_id", "id", "post_id") - metaculus_id = _str_or_none("metaculus_id", "post_id", "id") - return question_id, metaculus_id - - -def extract_records_from_question_dir( - question_dir: str, - *, - run_id: str | None = None, - bot: str | None = None, - question_id: str | None = None, - metaculus_id: str | None = None, - question_url: str | None = None, -) -> list[CitationRecord]: - """Aggregate CitationRecords across every trace in one ``q_*`` dir. - - Question provenance is read from ``question.json`` in the dir; pass any of - ``question_id`` / ``metaculus_id`` / ``question_url`` to override what's - found there (or to supply it when the file is absent). - """ - found_qid, found_mid = _read_question_metadata(question_dir) - question_id = question_id or found_qid - metaculus_id = metaculus_id or found_mid - if question_url is None and metaculus_id is not None: - question_url = METACULUS_QUESTION_URL_FMT.format(metaculus_id) - - records: list[CitationRecord] = [] - for trace_path in sorted(glob.glob(os.path.join(question_dir, "traces_*.jsonl"))): - records.extend( - extract_records_from_trace_file( - trace_path, - run_id=run_id, - bot=bot, - question_id=question_id, - metaculus_id=metaculus_id, - question_url=question_url, - ) - ) - return records - - -def _bot_name_from_dir(bot_dir: str) -> str: - """``.../bot_complex`` -> ``complex``.""" - name = os.path.basename(bot_dir) - return name[len("bot_") :] if name.startswith("bot_") else name - - -def _question_dirs_flat(run_dir: str) -> list[str]: - """Question dirs directly under ``run_dir`` (no ``bot_*`` level). - - A "question dir" is any immediate subdirectory that actually contains - ``traces_*.jsonl``. This handles flatter layouts (e.g. a backfill of one - bot's runs as ``//traces_*.jsonl``) where the ``bot_*`` - grouping is absent. - """ - dirs = [] - for entry in sorted(glob.glob(os.path.join(run_dir, "*"))): - if os.path.isdir(entry) and glob.glob(os.path.join(entry, "traces_*.jsonl")): - dirs.append(entry) - return dirs - - -def harvest_run( - run_dir: str, *, run_id: str | None = None, bot: str | None = None -) -> list[CitationRecord]: - """Build a citation manifest from a whole traced run directory. - - Primary layout is ``/bot_*/q_*/traces_*.jsonl``, deriving ``run_id`` - from the run dir's name and ``bot`` from each ``bot_*`` subdir. If no - ``bot_*`` subdirs exist, falls back to a **flat layout** — - ``//traces_*.jsonl`` — attributing every question to a - single bot (the ``bot`` argument, else the run dir's name). Question - provenance still comes from each dir's ``question.json``. - - Returns the flat list of CitationRecords (one per URL occurrence); feed it - through :func:`url_extraction.dedupe_records` before capture for one row per - URL. - """ - run_id = run_id or os.path.basename(os.path.normpath(run_dir)) - records: list[CitationRecord] = [] - - bot_dirs = sorted(glob.glob(os.path.join(run_dir, "bot_*"))) - if bot_dirs: - for bot_dir in bot_dirs: - bot_name = _bot_name_from_dir(bot_dir) - for question_dir in sorted(glob.glob(os.path.join(bot_dir, "q_*"))): - records.extend( - extract_records_from_question_dir( - question_dir, run_id=run_id, bot=bot_name - ) - ) - return records - - # Flat fallback: no bot_* grouping. One bot, question dirs directly below. - bot_name = bot or os.path.basename(os.path.normpath(run_dir)) - for question_dir in _question_dirs_flat(run_dir): - records.extend( - extract_records_from_question_dir(question_dir, run_id=run_id, bot=bot_name) - ) - return records diff --git a/forecasting_tools/agents_and_tools/source_archive/ingest/url_extraction.py b/forecasting_tools/agents_and_tools/source_archive/ingest/url_extraction.py deleted file mode 100644 index b8b06d3b..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/ingest/url_extraction.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Extract URLs from free text and markdown. - -Bots surface their sources as prose with embedded links (e.g. the reasoning -comment they post on a question). This module pulls those URLs out and turns -them into :class:`CitationRecord` provenance rows — the manifest that feeds the -capture pipeline. - -It handles markdown links ``[label](url)``, autolinks ````, and bare URLs, -and trims the trailing punctuation that so often clings to a URL in prose. -""" - -from __future__ import annotations - -import re -from collections.abc import Iterable - -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) -from forecasting_tools.agents_and_tools.source_archive.models import CitationRecord - -# Markdown link target: [label](url) or [label](), optionally with a title. -_MD_LINK = re.compile(r"\[[^\]]*\]\(\s*]+)>?[^)]*\)", re.IGNORECASE) -# Autolink: -_AUTOLINK = re.compile(r"<(https?://[^>\s]+)>", re.IGNORECASE) -# Bare URL. Parens are allowed in the match and removed by _trim only when -# unbalanced, so trailing prose parens drop but ``..._(disambiguation)`` survives. -_BARE = re.compile(r"(https?://[^\s<>\"'\]]+)", re.IGNORECASE) - -# Characters commonly stuck to the end of a URL in prose (incl. markdown-escape -# residue: a trailing backslash or backtick). -_TRAILING = ".,;:!?'\"\\`" - - -def _cut_markdown_tail(url: str) -> str: - """Cut a URL at a markdown reference/link tail the bare-URL scan can swallow. - - Bots sometimes emit ``…/story?id=123)[10](https://other…)`` where ``)[10](…`` - is a markdown reference glued onto a real URL. The leading ``)`` was never - part of the URL, so cut at the first ``)[`` or ``](`` boundary. - """ - cut = len(url) - for marker in (")[", "]("): - i = url.find(marker) - if i > 0: - cut = min(cut, i) - return url[:cut] - - -def _trim(url: str) -> str: - """Strip trailing punctuation, and a closing bracket/paren only when it is - unbalanced (so Wikipedia-style ``..._(disambiguation)`` URLs survive).""" - url = _cut_markdown_tail(url) - while url: - last = url[-1] - if last in _TRAILING: - url = url[:-1] - elif last == ")" and url.count("(") < url.count(")"): - url = url[:-1] - elif last == "]" and url.count("[") < url.count("]"): - url = url[:-1] - else: - break - return url - - -def extract_urls(text: str | None) -> list[str]: - """Return the distinct http(s) URLs in ``text``, in first-seen order. - - Distinctness is by *canonical* URL (see :func:`canonicalize_url`), so - ``…/x`` and ``…/x?utm_source=…`` count once; the original first-seen string - is returned. - """ - if not text: - return [] - seen: set[str] = set() - ordered: list[str] = [] - for pattern in (_MD_LINK, _AUTOLINK, _BARE): - for match in pattern.finditer(text): - url = _trim(match.group(1)) - if not url: - continue - key = canonicalize_url(url) - if key not in seen: - seen.add(key) - ordered.append(url) - return ordered - - -def extract_citation_records( - text: str | None, - *, - run_id: str | None = None, - bot: str | None = None, - question_id: str | None = None, - metaculus_id: str | None = None, - question_url: str | None = None, - comment_id: str | None = None, - trace: str | None = None, - tool_name: str | None = None, - origin: str | None = None, -) -> list[CitationRecord]: - """Extract URLs from ``text`` and wrap each in a CitationRecord with the - given provenance.""" - return [ - CitationRecord( - url=url, - run_id=run_id, - bot=bot, - question_id=question_id, - metaculus_id=metaculus_id, - question_url=question_url, - comment_id=comment_id, - trace=trace, - tool_name=tool_name, - origin=origin, - ) - for url in extract_urls(text) - ] - - -def dedupe_records(records: Iterable[CitationRecord]) -> list[CitationRecord]: - """Keep the first record per *canonical* URL, preserving order.""" - seen: set[str] = set() - out: list[CitationRecord] = [] - for r in records: - if not r.url: - continue - key = canonicalize_url(r.url) - if key not in seen: - seen.add(key) - out.append(r) - return out diff --git a/forecasting_tools/agents_and_tools/source_archive/layout.py b/forecasting_tools/agents_and_tools/source_archive/layout.py deleted file mode 100644 index b63c1435..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/layout.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Key layout for manifests and run reports in the blob store. - -Manifests and reports used to be written flat (``manifests/.jsonl``, -``reports/.json``), which turns into one giant folder as runs -accumulate. New writes nest by run family instead: - -- an explicit ``group`` pins the folder (e.g. ``sprints/myrun``); -- ``daily-YYYY-MM-DD`` run ids nest under ``daily//``; -- everything else lands under ``adhoc/``. - -Readers that list by prefix (catalog, coverage) pick up both layouts for free; -exact-key readers should try :func:`manifest_key_candidates` / -:func:`report_key_candidates` — nested first, then the legacy flat key — so -archives written before the nesting keep working. - -Keys here are store-relative (no ``s3_prefix``); callers prepend the prefix. -""" - -from __future__ import annotations - -import re - -_DAILY_RUN_ID = re.compile(r"^daily-(\d{4}-\d{2})-\d{2}$") - - -def _folder(run_id: str, group: str | None) -> str: - if group: - return group.strip("/") - match = _DAILY_RUN_ID.match(run_id) - if match: - return f"daily/{match.group(1)}" - return "adhoc" - - -def manifest_key(run_id: str, group: str | None = None) -> str: - """Nested key for a run's citation manifest.""" - return f"manifests/{_folder(run_id, group)}/{run_id}.jsonl" - - -def report_key(run_id: str, suffix: str, group: str | None = None) -> str: - """Nested key for a run report artifact (``suffix`` e.g. ``.json``).""" - return f"reports/{_folder(run_id, group)}/{run_id}{suffix}" - - -def legacy_manifest_key(run_id: str) -> str: - return f"manifests/{run_id}.jsonl" - - -def legacy_report_key(run_id: str, suffix: str) -> str: - return f"reports/{run_id}{suffix}" - - -def manifest_key_candidates(run_id: str, group: str | None = None) -> list[str]: - """Where a run's manifest may live, preferred (nested) first.""" - return [manifest_key(run_id, group), legacy_manifest_key(run_id)] - - -def report_key_candidates( - run_id: str, suffix: str, group: str | None = None -) -> list[str]: - """Where a run's report may live, preferred (nested) first.""" - return [report_key(run_id, suffix, group), legacy_report_key(run_id, suffix)] diff --git a/forecasting_tools/agents_and_tools/source_archive/manifest.py b/forecasting_tools/agents_and_tools/source_archive/manifest.py deleted file mode 100644 index 4b629fd1..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/manifest.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Per-run citation manifest: one JSONL record per (URL, citation). - -This is the provenance layer a bot emits and the input to the capture pipeline. -One manifest per run, stored in the blob store under the nested ``manifests/`` -layout (see :mod:`layout`); reads fall back to the legacy flat key. -""" - -from __future__ import annotations - -import json -from collections.abc import Iterable, Iterator -from pathlib import Path - -from forecasting_tools.agents_and_tools.source_archive import layout -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.models import CitationRecord -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) - - -def dumps(records: Iterable[CitationRecord]) -> str: - return "\n".join(json.dumps(r.model_dump(), sort_keys=True) for r in records) - - -def loads(text: str) -> list[CitationRecord]: - out: list[CitationRecord] = [] - for line in text.splitlines(): - line = line.strip() - if line: - out.append(CitationRecord.model_validate(json.loads(line))) - return out - - -def unique_urls(records: Iterable[CitationRecord]) -> Iterator[str]: - """Yield each distinct URL once, preserving first-seen order. - - Distinctness is by *canonical* URL (see :func:`canonicalize_url`), so - near-duplicate links collapse to a single fetch; the original first-seen URL - string is what's yielded, for provenance. - """ - seen: set[str] = set() - for r in records: - if not r.url: - continue - key = canonicalize_url(r.url) - if key not in seen: - seen.add(key) - yield r.url - - -# --- file io --------------------------------------------------------------- -def read_file(path: str | Path) -> list[CitationRecord]: - return loads(Path(path).read_text(encoding="utf-8")) - - -def write_file(path: str | Path, records: Iterable[CitationRecord]) -> None: - Path(path).write_text(dumps(records), encoding="utf-8") - - -# --- blob store io --------------------------------------------------------- -def manifest_key( - run_id: str, config: ArchiveConfig | None = None, group: str | None = None -) -> str: - prefix = (config or ArchiveConfig()).s3_prefix.rstrip("/") - return f"{prefix}/{layout.manifest_key(run_id, group)}" - - -def read_blob( - store: BlobStore, - run_id: str, - config: ArchiveConfig | None = None, - group: str | None = None, -) -> list[CitationRecord]: - prefix = (config or ArchiveConfig()).s3_prefix.rstrip("/") - candidates = [ - f"{prefix}/{k}" for k in layout.manifest_key_candidates(run_id, group) - ] - # Prefer the nested key; fall back to the legacy flat key for old runs. - for key in candidates[:-1]: - if store.exists(key): - return loads(store.get(key).decode("utf-8")) - return loads(store.get(candidates[-1]).decode("utf-8")) - - -def write_blob( - store: BlobStore, - run_id: str, - records: Iterable[CitationRecord], - config: ArchiveConfig | None = None, - group: str | None = None, -) -> None: - store.put( - manifest_key(run_id, config, group), - dumps(records).encode("utf-8"), - content_type="application/x-ndjson", - ) diff --git a/forecasting_tools/agents_and_tools/source_archive/models.py b/forecasting_tools/agents_and_tools/source_archive/models.py deleted file mode 100644 index 08c63cd6..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/models.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Core data structures shared across the source-archive pipeline.""" - -from __future__ import annotations - -import hashlib -from datetime import datetime, timezone -from typing import Any - -from pydantic import BaseModel, Field - -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) - - -def utcnow_iso() -> str: - return datetime.now(timezone.utc).isoformat() - - -def url_hash(url: str) -> str: - """Stable key for a URL — groups every capture of that URL together. - - The URL is canonicalized first (see :func:`canonicalize_url`) so trivially - different links — tracking params, a trailing slash, a ``#fragment``, - query-param order, host case — collapse onto one key instead of being - stored and counted as separate sources. - """ - return hashlib.sha256(canonicalize_url(url).encode("utf-8")).hexdigest() - - -def content_hash(html: str | bytes) -> str: - """Hash of page content — dedups identical re-fetches of the same URL.""" - data = html.encode("utf-8") if isinstance(html, str) else html - return hashlib.sha256(data).hexdigest() - - -class CaptureResult(BaseModel): - """What a fetcher returns for a single URL, before it is stored.""" - - url: str - final_url: str - status_code: int | None = None - html: str | None = None - markdown: str | None = None - screenshot: bytes | None = None - screenshot_content_type: str | None = None - fetcher: str = "" - fetched_at: str = Field(default_factory=utcnow_iso) - metadata: dict[str, Any] = Field(default_factory=dict) - - @property - def content_hash(self) -> str: - basis = self.html if self.html else (self.markdown or self.final_url) - return content_hash(basis) - - -class StoredCapture(BaseModel): - """Pointer to a stored capture in the object store.""" - - url: str - url_hash: str - content_hash: str - status_code: int | None = None - fetcher: str = "" - captured_at: str = Field(default_factory=utcnow_iso) - html_key: str | None = None - screenshot_key: str | None = None - markdown_key: str | None = None - # Set when this capture reuses another URL's blobs because the fetched - # content was byte-identical (cross-URL content dedup); holds that URL's hash. - content_alias_of: str | None = None - first_seen: str = Field(default_factory=utcnow_iso) - last_seen: str = Field(default_factory=utcnow_iso) - - -class CitationRecord(BaseModel): - """One provenance record per (URL, citation) a bot emitted in a run. - - This is the manifest schema: a run produces a JSONL file of these, which is - the input to the capture pipeline. Fields are deliberately generic so any - bot's trace/comment format can be mapped onto them. - """ - - url: str - run_id: str | None = None - bot: str | None = None - question_id: str | None = None - metaculus_id: str | None = None - question_url: str | None = None - comment_id: str | None = None # Metaculus comment the URL was cited in - trace: str | None = None - tool_name: str | None = None - origin: str | None = None - # Search provenance (populated by instrumented trace ingest, not comments): - query: str | None = None # the search query the bot ran, if known - tool_args: dict[str, Any] | None = None # full tool input (query + filters…) - first_seen: str = Field(default_factory=utcnow_iso) diff --git a/forecasting_tools/agents_and_tools/source_archive/pipeline.py b/forecasting_tools/agents_and_tools/source_archive/pipeline.py deleted file mode 100644 index e5564669..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/pipeline.py +++ /dev/null @@ -1,317 +0,0 @@ -"""Capture pipeline: turn a list of cited URLs into archived captures. - -For each unique URL: - - 1. :meth:`ContentStore.lookup` — within the TTL? cache hit, skip the fetch. - 2. ``fetcher.fetch`` — tiered Playwright -> Firecrawl, quality-gated. - 3. quality gate — junk (404 / block / thin) is not archived. - 4. :meth:`ContentStore.store` — write blobs (deduped by content hash). -""" - -from __future__ import annotations - -import logging -import threading -from collections.abc import Iterable - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.fetchers.base import ( - Fetcher, - FetchError, -) -from forecasting_tools.agents_and_tools.source_archive.manifest import unique_urls -from forecasting_tools.agents_and_tools.source_archive.models import ( - CitationRecord, - StoredCapture, -) -from forecasting_tools.agents_and_tools.source_archive.quality import evaluate - -logger = logging.getLogger(__name__) - -# "cache_hit" | "stored" | "deduped" | "quality_failed" | "error" -Status = str -_STATUSES = ("cache_hit", "stored", "deduped", "quality_failed", "error") - - -class CaptureOutcome(BaseModel): - url: str - status: Status - stored: StoredCapture | None = None - reason: str = "" - - -class PipelineSummary(BaseModel): - outcomes: list[CaptureOutcome] = [] - - def count(self, status: Status) -> int: - return sum(1 for o in self.outcomes if o.status == status) - - @property - def captures(self) -> dict[str, StoredCapture]: - return {o.url: o.stored for o in self.outcomes if o.stored is not None} - - def __str__(self) -> str: - body = ", ".join(f"{s}={self.count(s)}" for s in _STATUSES) - return f"PipelineSummary(total={len(self.outcomes)}, {body})" - - -class CapturePipeline: - def __init__(self, fetcher: Fetcher, content_store: ContentStore): - self.fetcher = fetcher - self.content_store = content_store - - def capture_url(self, url: str) -> CaptureOutcome: - cached = self.content_store.lookup(url) - if cached is not None: - return CaptureOutcome(url=url, status="cache_hit", stored=cached) - - try: - result = self.fetcher.fetch(url) - except FetchError as e: - logger.info("fetch error for %s: %s", url, e) - return CaptureOutcome(url=url, status="error", reason=str(e)) - except Exception as e: # never let one bad URL abort the whole run - logger.warning("unexpected error capturing %s: %s", url, e) - return CaptureOutcome(url=url, status="error", reason=f"unexpected: {e}") - - # Gate here so any fetcher is covered; the tiered fetcher also gates - # internally to decide fallback, but this is the authoritative check. - verdict = evaluate(result) - if not verdict.passed: - return CaptureOutcome( - url=url, status="quality_failed", reason=verdict.reason - ) - - store_result = self.content_store.store(result) - status = "stored" if store_result.created else "deduped" - return CaptureOutcome(url=url, status=status, stored=store_result.capture) - - def run(self, urls: Iterable[str]) -> PipelineSummary: - summary = PipelineSummary() - for url in urls: - summary.outcomes.append(self.capture_url(url)) - return summary - - def run_manifest(self, records: Iterable[CitationRecord]) -> PipelineSummary: - return self.run(unique_urls(records)) - - -# An outcome whose error reason contains one of these means the browser itself -# died (crash, OOM, or the machine slept and severed the CDP pipe) — not a -# problem with the URL. Without recovery, every later URL in that worker's shard -# would error against the dead browser, so we rebuild the browser and retry. -_DEAD_BROWSER_MARKERS = ( - "has been closed", - "Target page, context or browser", - "Browser.new_context", - "Connection closed", - "browser has been closed", -) - - -def _browser_died(reason: str | None) -> bool: - return any(m in (reason or "") for m in _DEAD_BROWSER_MARKERS) - - -def _close_quietly(cm, timeout_s: float = 15.0) -> None: - """Tear down a fetcher context manager, but never block on it: a wedged - browser's ``close()`` can itself hang, so run it in a daemon thread and give - up after ``timeout_s`` (the leftover process is reaped at the end of the run). - """ - done = threading.Event() - - def _close() -> None: - try: - cm.__exit__(None, None, None) - except Exception: - pass - finally: - done.set() - - threading.Thread(target=_close, daemon=True).start() - done.wait(timeout_s) - - -def _running_loop(): - import asyncio - - try: - return asyncio.get_running_loop() - except RuntimeError: - return None - - -def _restore_thread_loop_state(baseline) -> None: - """Un-poison a worker thread's asyncio state after abandoning a dead - sync-Playwright instance. - - Sync Playwright drives its asyncio loop *on the calling thread* (via a - greenlet), so while an instance is alive the thread is marked as being - inside a running event loop. A clean ``stop()`` clears that mark — but a - SIGKILLed browser can't be closed cleanly: ``close()``/``stop()`` fail or - hang, and both are thread-affine, so :func:`_close_quietly`'s helper thread - can't reach them either. The abandoned loop then stays registered as - running, and the next ``sync_playwright().start()`` on this thread refuses - with "Playwright Sync API inside the asyncio loop", killing the rebuild. - - Resetting the thread-local marker to its pre-fetcher ``baseline`` lets the - rebuilt fetcher start a fresh loop. The old loop object is leaked on - purpose (its browser processes are swept by the reaper); that is the price - of recovering without touching thread-affine Playwright internals. - """ - import asyncio - - if _running_loop() is baseline: - return - try: - asyncio.events._set_running_loop(baseline) - except Exception: - pass - - -def _reap_browser_descendants() -> None: - """Best-effort: kill automation Chromium descending from this process. Used - both to recover a wedged worker (kill its browser so the blocked sync call - errors out) and to sweep leftovers at end of run. No-op without psutil so it - never becomes a hard dependency. - """ - try: - import os - - import psutil - except Exception: - return - try: - for child in psutil.Process(os.getpid()).children(recursive=True): - try: - if "chrom" in (child.name() or "").lower(): - child.kill() - except Exception: - pass - except Exception: - pass - - -def capture_urls_concurrent( - urls: Iterable[str], - store: ContentStore, - config, - fetcher_factory, - per_url_timeout: float | None = None, - reaper=_reap_browser_descendants, -) -> PipelineSummary: - """Capture ``urls`` across ``config.concurrency`` worker threads. - - Headless Chromium's sync API is **thread-affine** — a browser must be used on - the thread that created it — so each worker opens its **own** browser via - ``fetcher_factory(config)`` and runs all captures inline on its own thread. - The content store is shared (writes are keyed by URL hash and idempotent, so - shards never collide). Order of outcomes is not preserved. - - Hang protection runs *out of band*: a supervisor thread watches each worker's - heartbeat and, if one is stuck on a single URL past ``per_url_timeout`` (a - wedged sync call whose Playwright timeout never fires — e.g. the machine - slept and severed the CDP pipe), it **kills the browser processes**. That is - an OS-level action (safe across threads, unlike touching Playwright objects), - so the blocked call errors out and the worker rebuilds via the same - dead-browser path — no single stuck worker can freeze the whole run. - """ - import time - from concurrent.futures import ThreadPoolExecutor - - url_list = list(urls) - workers = max(1, int(getattr(config, "concurrency", 1) or 1)) - if per_url_timeout is None: - nav_s = float(getattr(config, "nav_timeout_ms", 30000)) / 1000.0 - per_url_timeout = max(90.0, nav_s * 4) - - # worker index -> monotonic start of its current URL (None when between URLs) - heartbeats: dict[int, float | None] = {} - hb_lock = threading.Lock() - stop = threading.Event() - - def supervisor() -> None: - interval = max(0.5, min(per_url_timeout / 2, 30.0)) - while not stop.wait(interval): - now = time.monotonic() - with hb_lock: - stalled = [ - w - for w, t in heartbeats.items() - if t is not None and now - t > per_url_timeout - ] - if stalled: - logger.warning( - "worker(s) %s stuck > %.0fs on one URL; killing browsers to recover", - stalled, - per_url_timeout, - ) - reaper() - with hb_lock: # grace: don't reap again before workers rebuild - for w in list(heartbeats): - if heartbeats[w] is not None: - heartbeats[w] = now - - def work(idx: int, shard: list[str]) -> list[CaptureOutcome]: - outcomes: list[CaptureOutcome] = [] - baseline_loop = _running_loop() # almost always None; see _restore_... - cm = fetcher_factory(config) - pipeline = CapturePipeline(cm.__enter__(), store) - try: - for url in shard: - with hb_lock: - heartbeats[idx] = time.monotonic() - outcome = pipeline.capture_url(url) - if outcome.status == "error" and _browser_died(outcome.reason): - logger.warning( - "browser died; rebuilding worker %d, retrying %s", idx, url - ) - _close_quietly(cm) - _restore_thread_loop_state(baseline_loop) - try: - cm = fetcher_factory(config) - pipeline = CapturePipeline(cm.__enter__(), store) - except Exception as e: - # A failed rebuild must never kill the run: keep this - # URL's error outcome and move on — the still-dead - # pipeline makes the next URL error with a dead-browser - # reason, which retries the rebuild. - logger.warning( - "worker %d rebuild failed (%s); keeping error outcome", - idx, - e, - ) - else: - with hb_lock: - heartbeats[idx] = time.monotonic() - # one retry on a fresh browser - outcome = pipeline.capture_url(url) - outcomes.append(outcome) - with hb_lock: - heartbeats[idx] = None - finally: - _close_quietly(cm) - _restore_thread_loop_state(baseline_loop) - return outcomes - - supervisor_thread = threading.Thread(target=supervisor, daemon=True) - supervisor_thread.start() - try: - if workers == 1: - heartbeats[0] = None - return PipelineSummary(outcomes=work(0, url_list)) - - shards = [url_list[i::workers] for i in range(workers)] - for i in range(workers): - heartbeats[i] = None - summary = PipelineSummary() - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = [pool.submit(work, i, shards[i]) for i in range(workers)] - for future in futures: - summary.outcomes.extend(future.result()) - return summary - finally: - stop.set() - reaper() diff --git a/forecasting_tools/agents_and_tools/source_archive/quality.py b/forecasting_tools/agents_and_tools/source_archive/quality.py deleted file mode 100644 index 0bed3497..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/quality.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Quality gate for captures. - -A headless browser will happily "succeed" at screenshotting a 404 or a bot-block -interstitial. Gate captures on HTTP status, content length, and block-page -signatures before archiving, so junk is neither stored nor counted as a success -(and so the tiered fetcher knows when to fall back to another backend). -""" - -from __future__ import annotations - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive.models import CaptureResult - -# Substrings that strongly indicate a block / interstitial rather than the real -# page. Matched case-insensitively against extracted text. -BLOCK_SIGNATURES = ( - "verify you are a human", - "are you a human", - "checking your browser before", - "enable javascript and cookies to continue", - "please enable javascript", - "access to this page has been denied", - "access denied", - "request unsuccessful. incapsula", - "attention required! | cloudflare", - "ddos protection by cloudflare", - "ray id:", - "captcha", - "unusual traffic from your computer", -) - -MIN_TEXT_LEN = 200 - - -class QualityVerdict(BaseModel): - passed: bool - reason: str = "" - - -def evaluate( - result: CaptureResult, *, min_text_len: int = MIN_TEXT_LEN -) -> QualityVerdict: - if result.status_code is not None and result.status_code >= 400: - return QualityVerdict(passed=False, reason=f"http_status={result.status_code}") - - text = (result.markdown or result.html or "").strip() - if len(text) < min_text_len: - return QualityVerdict(passed=False, reason=f"thin_content len={len(text)}") - - lowered = text.lower() - for sig in BLOCK_SIGNATURES: - if sig in lowered: - return QualityVerdict(passed=False, reason=f"block_signature={sig!r}") - - return QualityVerdict(passed=True) diff --git a/forecasting_tools/agents_and_tools/source_archive/reindex.py b/forecasting_tools/agents_and_tools/source_archive/reindex.py deleted file mode 100644 index 0a1e5a45..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/reindex.py +++ /dev/null @@ -1,278 +0,0 @@ -"""One-off reindex / dedup audit for an existing archive. - -This walks the canonical per-URL indexes already in a store and reports how much -the smarter-dedup work (see ``ROADMAP.md`` Plan 1) would collapse, **without -mutating anything by default**. It answers the practical question: *after exact -canonicalization and content dedup, are there still many URLs that look like the -same page?* — i.e. whether the fuzzy near-dup phase (D) is worth building. - -Three lenses: - - - **Canonicalization (Phase A):** group stored URLs by :func:`canonicalize_url`. - Any group with >1 distinct raw URL is a set that *now* shares one key. - - **Content (Phase C):** group distinct canonical URLs by their latest content - hash. A group with >1 URL is byte-identical pages reachable at different URLs. - - **Near-dup signal (Phase D candidate):** of the URLs surviving both dedups, - group by ``scheme://host/path`` ignoring the query string. Big groups mean - "same path, differing query" pages that exact dedup leaves separate — the - cases fuzzy matching would target. - -Run it:: - - # against the configured S3 bucket (read-only audit) - WEB_ARCHIVE_S3_BUCKET=my-web-archive WEB_ARCHIVE_AWS_PROFILE=default \\ - python -m forecasting_tools.agents_and_tools.source_archive.reindex - - # against a local capture dir - python -m forecasting_tools.agents_and_tools.source_archive.reindex --local ./archive - - # additionally (re)build the content reverse index for existing captures - python -m forecasting_tools.agents_and_tools.source_archive.reindex --apply - -``--apply`` only writes the additive ``index/by-content/`` reverse index (safe, -idempotent). It does **not** move blobs or re-key the per-URL indexes; that -heavier migration is intentionally deferred (the archive is young). -""" - -from __future__ import annotations - -import argparse -import json -import sys -from collections import defaultdict -from urllib.parse import urlsplit - -from pydantic import BaseModel - -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.content_store import ContentStore -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) - - -class Cluster(BaseModel): - key: str - urls: list[str] - - -class AnalysisReport(BaseModel): - total_url_indexes: int = 0 - alias_indexes: int = 0 # already-collapsed redirects (Phase B) - canonical_captures: int = 0 # distinct stored URLs with content - distinct_after_canonicalization: int = 0 - distinct_after_content_dedup: int = 0 - canonicalization_clusters: list[Cluster] = [] # raw URLs that now share a key - content_clusters: list[Cluster] = [] # different URLs, identical content - near_dup_clusters: list[Cluster] = [] # same host+path, differing query - - def __str__(self) -> str: - merged_a = sum(len(c.urls) - 1 for c in self.canonicalization_clusters) - merged_c = sum(len(c.urls) - 1 for c in self.content_clusters) - lines = [ - "Source-archive dedup audit", - "=" * 40, - f"URL indexes scanned : {self.total_url_indexes}", - f" of which alias (redirect) : {self.alias_indexes}", - f" of which canonical capture : {self.canonical_captures}", - "", - f"Distinct URLs (raw) : {self.canonical_captures}", - f"After canonicalization (A) : {self.distinct_after_canonicalization}" - f" (−{merged_a} merged)", - f"After content dedup (C) : {self.distinct_after_content_dedup}" - f" (−{merged_c} byte-identical)", - "", - f"Canonicalization clusters : {len(self.canonicalization_clusters)}", - f"Identical-content clusters : {len(self.content_clusters)}", - f"Near-dup candidates (D) : {len(self.near_dup_clusters)}" - " (same host+path, differing query)", - ] - - def _show(title: str, clusters: list[Cluster], limit: int = 5) -> None: - if not clusters: - return - lines.append("") - lines.append(f"--- top {title} ---") - for c in sorted(clusters, key=lambda x: len(x.urls), reverse=True)[:limit]: - lines.append(f" [{len(c.urls)}] {c.key}") - for u in c.urls[:4]: - lines.append(f" {u}") - if len(c.urls) > 4: - lines.append(f" … +{len(c.urls) - 4} more") - - _show("canonicalization clusters", self.canonicalization_clusters) - _show("identical-content clusters", self.content_clusters) - _show("near-dup candidates (Phase D signal)", self.near_dup_clusters) - return "\n".join(lines) - - -def _host_path(url: str) -> str: - parts = urlsplit(canonicalize_url(url)) - return f"{parts.scheme}://{parts.netloc}{parts.path}" - - -def iter_url_indexes(store: BlobStore, prefix: str): - """Yield ``(key, index_dict)`` for each per-URL index, skipping the reverse - content index under ``index/by-content/``.""" - index_prefix = f"{prefix.rstrip('/')}/index/" - content_sub = f"{index_prefix}by-content/" - for key in store.list_keys(index_prefix): - if not key.endswith(".json") or key.startswith(content_sub): - continue - try: - yield key, json.loads(store.get(key).decode("utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError): - continue - - -def analyze(store: BlobStore, config: ArchiveConfig) -> AnalysisReport: - report = AnalysisReport() - by_canonical: dict[str, list[str]] = defaultdict(list) - by_content: dict[str, list[str]] = defaultdict(list) - - for _key, index in iter_url_indexes(store, config.s3_prefix): - report.total_url_indexes += 1 - if index.get("alias_of"): - report.alias_indexes += 1 - continue - url = index.get("url") - if not url or not index.get("captures"): - continue - report.canonical_captures += 1 - by_canonical[canonicalize_url(url)].append(url) - ch = index.get("latest_content_hash") - if ch: - by_content[ch].append(url) - - report.distinct_after_canonicalization = len(by_canonical) - report.canonicalization_clusters = [ - Cluster(key=k, urls=sorted(set(v))) - for k, v in by_canonical.items() - if len(set(v)) > 1 - ] - - # Content dedup operates on the canonicalized URL set. - content_groups = {k: sorted(set(v)) for k, v in by_content.items()} - report.content_clusters = [ - Cluster(key=k, urls=v) for k, v in content_groups.items() if len(v) > 1 - ] - # distinct pages after content dedup = canonical URLs minus those merged away - merged_by_content = sum(len(v) - 1 for v in content_groups.values() if len(v) > 1) - report.distinct_after_content_dedup = max( - 0, report.distinct_after_canonicalization - merged_by_content - ) - - # Phase D signal: among canonical URLs, same host+path but differing query. - survivors = {canonicalize_url(u) for grp in by_canonical.values() for u in grp} - by_host_path: dict[str, set[str]] = defaultdict(set) - for u in survivors: - by_host_path[_host_path(u)].add(u) - report.near_dup_clusters = [ - Cluster(key=k, urls=sorted(v)) for k, v in by_host_path.items() if len(v) > 1 - ] - return report - - -def rebuild_content_index( - store: BlobStore, config: ArchiveConfig, *, apply: bool -) -> int: - """(Re)build ``index/by-content/`` from existing captures. Returns the number - of content groups (that would be) written. Additive and idempotent.""" - cstore = ContentStore(store, config) - groups: dict[str, list[tuple[str, str]]] = defaultdict(list) - for _key, index in iter_url_indexes(store, config.s3_prefix): - if index.get("alias_of") or not index.get("captures"): - continue - uh = index.get("url_hash") - url = index.get("url") - ch = index.get("latest_content_hash") - if uh and url and ch: - groups[ch].append((uh, url)) - - written = 0 - for ch, members in groups.items(): - written += 1 - if not apply: - continue - owner_uh, owner_url = members[0] - # Re-register every member; the first becomes canonical owner. - for uh, url in members: - blob_keys = None - if uh == owner_uh: - cap = index_blob_keys(store, config, owner_uh, ch) - blob_keys = cap - cstore._register_content(ch, uh, url, blob_keys) - return written - - -def index_blob_keys( - store: BlobStore, config: ArchiveConfig, uh: str, ch: str -) -> dict | None: - cstore = ContentStore(store, config) - index = cstore._read_index(uh) - if not index: - return None - cap = (index.get("captures") or {}).get(ch) - if not cap: - return None - return { - "html": cap.get("html_key"), - "markdown": cap.get("markdown_key"), - "screenshot": cap.get("screenshot_key"), - } - - -def _build_store(local_dir: str | None, bucket: str | None, config: ArchiveConfig): - if local_dir: - from forecasting_tools.agents_and_tools.source_archive.storage import ( - LocalBlobStore, - ) - - return LocalBlobStore(local_dir) - bucket = bucket or config.s3_bucket - if not bucket: - sys.exit( - "No S3 bucket configured. Set WEB_ARCHIVE_S3_BUCKET (or pass --bucket), " - "or use --local DIR." - ) - from forecasting_tools.agents_and_tools.source_archive.storage import S3BlobStore - - return S3BlobStore(bucket, config=config) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - prog="source-archive-reindex", - description="Audit (and optionally rebuild) dedup structures for an " - "existing archive.", - ) - parser.add_argument("--local", metavar="DIR", help="audit a local capture dir") - parser.add_argument("--bucket", help="override WEB_ARCHIVE_S3_BUCKET") - parser.add_argument( - "--apply", - action="store_true", - help="rebuild index/by-content/ for existing captures (additive)", - ) - parser.add_argument("--json", action="store_true", help="emit the report as JSON") - args = parser.parse_args(argv) - - config = ArchiveConfig.from_env() - store = _build_store(args.local, args.bucket, config) - - report = analyze(store, config) - if args.json: - print(report.model_dump_json(indent=2)) - else: - print(report) - - if args.apply: - n = rebuild_content_index(store, config, apply=True) - print(f"\nRebuilt index/by-content/ for {n} content group(s).") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/forecasting_tools/agents_and_tools/source_archive/reports.py b/forecasting_tools/agents_and_tools/source_archive/reports.py deleted file mode 100644 index 8ad95945..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/reports.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Persist each capture run's per-URL outcomes under ``reports/`` (nested per -:mod:`layout`; reads also pick up legacy flat ``reports/.json`` keys). - -The coverage report's job is to surface sources we should be collecting. A cited -source we have not archived falls into two very different buckets: - -- **never fetched** — it was harvested into a manifest but no capture run ever - attempted it. This is the real "we should go collect this" signal. -- **fetched but failed** — we tried and the fetch/quality gate rejected it - (Cloudflare, PDF, 404…). A capture problem, not a collection problem. - -Without persisted run outcomes the two are indistinguishable. Writing each run's -outcomes here lets coverage tell them apart. -""" - -from __future__ import annotations - -import json - -from forecasting_tools.agents_and_tools.source_archive import layout -from forecasting_tools.agents_and_tools.source_archive.canonicalize import ( - canonicalize_url, -) -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) - -CAPTURED_STATUSES = {"stored", "deduped", "cache_hit"} -FAILED_STATUSES = {"quality_failed", "error"} - - -def report_key(run_id: str, config: ArchiveConfig, group: str | None = None) -> str: - prefix = config.s3_prefix.rstrip("/") - return f"{prefix}/{layout.report_key(run_id, '.json', group)}" - - -def write_run_report( - store: BlobStore, - run_id: str, - summary, - config: ArchiveConfig, - group: str | None = None, -) -> str: - """Persist a run's per-URL outcomes; ``summary`` is a ``PipelineSummary``. - - ``backend`` is the fetcher that produced the capture (from the stored - capture, so it is set for stored/deduped/cache_hit outcomes and ``""`` - when unknown, e.g. errors) — it enables per-domain cost attribution. - """ - rows = [ - { - "url": o.url, - "status": o.status, - "reason": getattr(o, "reason", ""), - "backend": o.stored.fetcher if o.stored is not None else "", - } - for o in summary.outcomes - ] - key = report_key(run_id, config, group) - store.put( - key, json.dumps(rows, indent=2).encode("utf-8"), content_type="application/json" - ) - return key - - -def read_outcomes(store: BlobStore, config: ArchiveConfig) -> dict[str, str]: - """Map canonical URL -> last known capture status across all run reports. - - A captured status wins over a failed one (if we ever succeeded, that's the - truth). Returns ``{}`` if no reports exist yet. - """ - prefix = config.s3_prefix.rstrip("/") - out: dict[str, str] = {} - for key in store.list_keys(f"{prefix}/reports/"): - # reports/ also holds per-run cost breakdowns (``_cost.json``, - # a dict) — only run reports (a list of outcome rows) belong here. - if not key.endswith(".json") or key.endswith("_cost.json"): - continue - try: - rows = json.loads(store.get(key).decode("utf-8")) - except (UnicodeDecodeError, ValueError): - continue - if not isinstance(rows, list): - continue - for r in rows: - url = canonicalize_url(r.get("url", "")) - status = r.get("status", "") - if not url: - continue - if url not in out or status in CAPTURED_STATUSES: - out[url] = status - return out diff --git a/forecasting_tools/agents_and_tools/source_archive/storage/__init__.py b/forecasting_tools/agents_and_tools/source_archive/storage/__init__.py deleted file mode 100644 index a7c7755a..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/storage/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Blob storage backends for the source archive.""" - -from forecasting_tools.agents_and_tools.source_archive.storage.blob_store import ( - BlobStore, -) -from forecasting_tools.agents_and_tools.source_archive.storage.local_store import ( - LocalBlobStore, -) -from forecasting_tools.agents_and_tools.source_archive.storage.s3_store import ( - S3BlobStore, -) - -__all__ = ["BlobStore", "LocalBlobStore", "S3BlobStore"] diff --git a/forecasting_tools/agents_and_tools/source_archive/storage/blob_store.py b/forecasting_tools/agents_and_tools/source_archive/storage/blob_store.py deleted file mode 100644 index 7553c972..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/storage/blob_store.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Blob store interface. - -The content store and manifest layer depend on this abstraction, not on S3 -directly, so they can run offline against :class:`LocalBlobStore`. -""" - -from __future__ import annotations - -from collections.abc import Iterable -from typing import Protocol, runtime_checkable - - -@runtime_checkable -class BlobStore(Protocol): - def put( - self, key: str, data: bytes, *, content_type: str | None = None - ) -> None: ... - - def get(self, key: str) -> bytes: ... - - def exists(self, key: str) -> bool: ... - - def list_keys(self, prefix: str = "") -> Iterable[str]: - """Yield every stored key beginning with ``prefix`` (for reindex/audit).""" - ... diff --git a/forecasting_tools/agents_and_tools/source_archive/storage/local_store.py b/forecasting_tools/agents_and_tools/source_archive/storage/local_store.py deleted file mode 100644 index d85b0b0b..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/storage/local_store.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Filesystem-backed blob store for tests, local dev, and dry runs.""" - -from __future__ import annotations - -from pathlib import Path - - -class LocalBlobStore: - def __init__(self, root: str | Path): - self.root = Path(root) - - def _path(self, key: str) -> Path: - return self.root / key - - def put(self, key: str, data: bytes, *, content_type: str | None = None) -> None: - path = self._path(key) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - - def get(self, key: str) -> bytes: - return self._path(key).read_bytes() - - def exists(self, key: str) -> bool: - return self._path(key).exists() - - def list_keys(self, prefix: str = "") -> list[str]: - if not self.root.exists(): - return [] - keys = [ - p.relative_to(self.root).as_posix() - for p in self.root.rglob("*") - if p.is_file() - ] - return sorted(k for k in keys if k.startswith(prefix)) diff --git a/forecasting_tools/agents_and_tools/source_archive/storage/s3_store.py b/forecasting_tools/agents_and_tools/source_archive/storage/s3_store.py deleted file mode 100644 index 10914b94..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/storage/s3_store.py +++ /dev/null @@ -1,66 +0,0 @@ -"""S3-backed blob store (boto3). - -Bucket and credentials come from :class:`ArchiveConfig` / the environment and are -never hardcoded, so this is safe to publish. boto3 is optional and imported -lazily (``pip install forecasting-tools[source-archive]``). -""" - -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive.config import ArchiveConfig - - -class S3BlobStore: - def __init__( - self, bucket: str, *, config: ArchiveConfig | None = None, client=None - ): - if not bucket: - raise ValueError( - "S3BlobStore requires a bucket name (set WEB_ARCHIVE_S3_BUCKET)" - ) - self.bucket = bucket - self._config = config or ArchiveConfig() - self._client = client - - def _get_client(self): - if self._client is None: - try: - import boto3 - except ImportError as e: - raise ImportError( - "boto3 is not installed. Install it with " - "`pip install forecasting-tools[source-archive]`." - ) from e - - session = boto3.Session( - profile_name=self._config.aws_profile, - region_name=self._config.aws_region, - ) - self._client = session.client("s3") - return self._client - - def put(self, key: str, data: bytes, *, content_type: str | None = None) -> None: - extra = {"ContentType": content_type} if content_type else {} - self._get_client().put_object(Bucket=self.bucket, Key=key, Body=data, **extra) - - def get(self, key: str) -> bytes: - resp = self._get_client().get_object(Bucket=self.bucket, Key=key) - return resp["Body"].read() - - def exists(self, key: str) -> bool: - from botocore.exceptions import ClientError - - try: - self._get_client().head_object(Bucket=self.bucket, Key=key) - return True - except ClientError as e: - code = e.response.get("Error", {}).get("Code") - if code in ("404", "NoSuchKey", "NotFound"): - return False - raise - - def list_keys(self, prefix: str = ""): - paginator = self._get_client().get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): - for obj in page.get("Contents", []): - yield obj["Key"] diff --git a/forecasting_tools/agents_and_tools/source_archive/viewer.py b/forecasting_tools/agents_and_tools/source_archive/viewer.py deleted file mode 100644 index 022e632c..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/viewer.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Streamlit viewer for the source archive. - -Browse what the capture pipeline stored in S3: pick a captured URL and see its -**screenshot, markdown, and HTML** side by side, with the question/bot it came -from. Reads provenance from the run manifests and resolves each URL's latest -capture through its per-URL index — no local file wrangling. - -Run it:: - - # uses the same env as the rest of the archive (WEB_ARCHIVE_S3_BUCKET, etc.) - AWS_PROFILE=default WEB_ARCHIVE_S3_BUCKET=my-web-archive \\ - streamlit run forecasting_tools/agents_and_tools/source_archive/viewer.py - -Nothing here is deployment-specific: bucket/prefix/profile come from -``ArchiveConfig.from_env()``. -""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -# `streamlit run ` puts only the script's own directory on sys.path, not -# the repo root — so make `import forecasting_tools` work whether the package is -# pip-installed or just checked out. (viewer.py -> source_archive -> agents_and_tools -# -> forecasting_tools -> .) -_REPO_ROOT = Path(__file__).resolve().parents[3] -if str(_REPO_ROOT) not in sys.path: - sys.path.insert(0, str(_REPO_ROOT)) - -import pandas as pd # noqa: E402 -import streamlit as st # noqa: E402 - -from forecasting_tools.agents_and_tools.source_archive.config import ( # noqa: E402 - ArchiveConfig, -) -from forecasting_tools.agents_and_tools.source_archive.models import ( # noqa: E402 - url_hash, -) - -# --- S3 access (cached) ---------------------------------------------------- - - -@st.cache_resource(show_spinner=False) -def _client(profile: str | None, region: str | None): - import boto3 - - return boto3.Session( - profile_name=profile or None, region_name=region or None - ).client("s3") - - -def _cfg() -> ArchiveConfig: - return ArchiveConfig.from_env() - - -@st.cache_data(show_spinner=False) -def _list_keys(bucket: str, prefix: str) -> list[str]: - cfg = _cfg() - if cfg.local_dir: # filesystem-backed archive — list matching files - root = Path(cfg.local_dir) - if not root.exists(): - return [] - return [ - p.relative_to(root).as_posix() - for p in root.rglob("*") - if p.is_file() and p.relative_to(root).as_posix().startswith(prefix) - ] - s3 = _client(cfg.aws_profile, cfg.aws_region) - keys: list[str] = [] - token = None - while True: - kw = {"Bucket": bucket, "Prefix": prefix} - if token: - kw["ContinuationToken"] = token - resp = s3.list_objects_v2(**kw) - keys.extend(o["Key"] for o in resp.get("Contents", [])) - if not resp.get("IsTruncated"): - break - token = resp.get("NextContinuationToken") - return keys - - -@st.cache_data(show_spinner=False) -def _get_bytes(bucket: str, key: str) -> bytes | None: - cfg = _cfg() - if cfg.local_dir: - p = Path(cfg.local_dir) / key - return p.read_bytes() if p.exists() else None - s3 = _client(cfg.aws_profile, cfg.aws_region) - try: - return s3.get_object(Bucket=bucket, Key=key)["Body"].read() - except Exception: - return None - - -# Metaculus question id -> review URL. Derived at display time (not stored) so -# there's no redundant, drift-prone URL column in S3. -_METACULUS_QUESTION_BASE = "https://www.metaculus.com/questions/" - - -def _metaculus_url(metaculus_id) -> str: - if metaculus_id in (None, "", "null"): - return "" - return f"{_METACULUS_QUESTION_BASE}{metaculus_id}/" - - -def _comment_url(metaculus_id, comment_id) -> str: - """Deep-link to the specific comment the URL was cited in.""" - base = _metaculus_url(metaculus_id) - if not base or comment_id in (None, "", "null"): - return "" - return f"{base}#comment-{comment_id}" - - -@st.cache_data(show_spinner="Loading manifests…") -def _manifest_rows(bucket: str, prefix: str) -> pd.DataFrame: - """Every (question, bot, url) the bots cited, from the run manifests.""" - rows = [] - for key in _list_keys(bucket, f"{prefix}/manifests/"): - body = _get_bytes(bucket, key) - if not body: - continue - for line in body.decode("utf-8").splitlines(): - line = line.strip() - if not line: - continue - r = json.loads(line) - rows.append( - { - "question": r.get("question_id") or "(none)", - "bot": r.get("bot") or "(none)", - "run_id": r.get("run_id") or "", - "origin": r.get("origin") or "", - "query": r.get("query") or "", - "metaculus": _metaculus_url(r.get("metaculus_id")), - "comment": _comment_url(r.get("metaculus_id"), r.get("comment_id")), - "url": r.get("url", ""), - "question_url": r.get("question_url") or "", - "tool_args": r.get("tool_args"), - } - ) - df = pd.DataFrame(rows) - if not df.empty: - # Keep distinct provenance (a URL cited via two origins/runs = two rows). - df = df.drop_duplicates( - subset=["question", "bot", "run_id", "origin", "url"] - ).reset_index(drop=True) - return df - - -def _scrape_report(bucket: str, prefix: str, view: pd.DataFrame): - """Per-question scraping cost: which backend captured each URL. - - Self-hosted Playwright is free; Firecrawl (the fallback) costs ~1 credit per - page and is what actually accrues spend once a key is configured. We classify - each *stored* capture by ``fetcher`` and count Firecrawl pages per question. - - Caveat: only successful captures are recorded in the index, so a Firecrawl - attempt that failed its quality gate isn't counted here — billed attempts - aren't yet persisted (see the note in the UI). - """ - per_q: dict[str, dict] = {} - for _, row in view.iterrows(): - cap = _index(bucket, prefix, row["url"]) - q = row["question"] - agg = per_q.setdefault( - q, - { - "question": q, - "urls": 0, - "captured": 0, - "playwright": 0, - "firecrawl": 0, - "other": 0, - }, - ) - agg["urls"] += 1 - if not cap: - continue - agg["captured"] += 1 - fetcher = (cap.get("fetcher") or "").lower() - if fetcher in ("playwright", "firecrawl"): - agg[fetcher] += 1 - else: - agg["other"] += 1 - return per_q - - -@st.cache_data(show_spinner=False) -def _index(bucket: str, prefix: str, url: str) -> dict | None: - """Latest stored capture for a URL (keys + metadata), or None if uncaptured.""" - body = _get_bytes(bucket, f"{prefix}/index/{url_hash(url)}.json") - if not body: - return None - idx = json.loads(body.decode("utf-8")) - ch = idx.get("latest_content_hash") - cap = (idx.get("captures") or {}).get(ch) - return cap - - -# --- UI -------------------------------------------------------------------- - - -def main() -> None: - st.set_page_config(page_title="Source Archive Viewer", layout="wide") - cfg = _cfg() - st.title("📚 Source Archive Viewer") - - location = cfg.local_dir or cfg.s3_bucket - if not location: - st.error( - "No archive configured. Set WEB_ARCHIVE_LOCAL_DIR (a local capture " - "directory) or WEB_ARCHIVE_S3_BUCKET (S3), then reload." - ) - st.stop() - if cfg.local_dir: - st.caption(f"📂 local: {cfg.local_dir}/{cfg.s3_prefix}") - else: - st.caption( - f"s3://{cfg.s3_bucket}/{cfg.s3_prefix} · " - f"profile={cfg.aws_profile or 'default'}" - ) - - with st.sidebar: - st.header("Filters") - if st.button("🔄 Refresh"): - st.cache_data.clear() - st.rerun() - - df = _manifest_rows(location, cfg.s3_prefix) - if df.empty: - st.warning("No manifests found under this prefix yet. Run a capture first.") - st.stop() - - with st.sidebar: - bots = sorted(df["bot"].unique()) - qs = sorted(df["question"].unique()) - sel_bots = st.multiselect("Bot", bots, default=bots) - sel_qs = st.multiselect("Question", qs, default=qs) - search = st.text_input("URL contains") - - view = df[df["bot"].isin(sel_bots) & df["question"].isin(sel_qs)] - if search: - view = view[view["url"].str.contains(search, case=False, na=False)] - view = view.reset_index(drop=True) - - st.subheader(f"{len(view)} cited URL(s)") - - # Resolve capture status for the filtered rows (cached per-URL). - if len(view) > 300: - st.info( - "Showing 300 of %d — narrow with the filters for capture details." - % len(view) - ) - table = [] - for _, row in view.head(300).iterrows(): - cap = _index(location, cfg.s3_prefix, row["url"]) - table.append( - { - "question": row["question"], - "bot": row["bot"], - "run_id": row["run_id"], - "origin": row["origin"], - "captured": "✅" if cap else "—", - "fetcher": (cap or {}).get("fetcher", ""), - "captured_at": (cap or {}).get("captured_at", "")[:19], - "metaculus": row["metaculus"], - "comment": row["comment"], - "url": row["url"], - } - ) - st.dataframe( - pd.DataFrame(table), - use_container_width=True, - hide_index=True, - column_config={ - # Show the full link address as the clickable text (not a label). - "url": st.column_config.LinkColumn("url"), - "metaculus": st.column_config.LinkColumn( - "metaculus", display_text="question ↗" - ), - "comment": st.column_config.LinkColumn("comment", display_text="comment ↗"), - }, - ) - - if st.sidebar.checkbox("💸 Show scraping cost"): - st.subheader("💸 Scraping cost (filtered set)") - rate = st.number_input( - "Firecrawl cost per page ($)", - min_value=0.0, - value=0.001, - step=0.0005, - format="%.4f", - help="Self-hosted Playwright is free; this prices the Firecrawl " - "fallback. Adjust to your plan's credit rate.", - ) - per_q = _scrape_report(location, cfg.s3_prefix, view.head(300)) - rows, t_fc, t_pw, t_cap, t_url = [], 0, 0, 0, 0 - for agg in sorted(per_q.values(), key=lambda a: a["question"]): - rows.append( - { - "question": agg["question"], - "urls": agg["urls"], - "captured": agg["captured"], - "playwright (free)": agg["playwright"], - "firecrawl (paid)": agg["firecrawl"], - "firecrawl $": round(agg["firecrawl"] * rate, 4), - } - ) - t_fc += agg["firecrawl"] - t_pw += agg["playwright"] - t_cap += agg["captured"] - t_url += agg["urls"] - st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True) - a, b, c = st.columns(3) - a.metric("Captured", f"{t_cap}/{t_url}") - b.metric("Firecrawl pages", t_fc, help="Playwright pages are free") - c.metric("Est. Firecrawl cost", f"${t_fc * rate:.4f}") - st.caption( - f"Playwright (free): {t_pw} · Firecrawl (paid): {t_fc}. " - "⚠️ Only **successful** captures carry a fetcher in the index, so " - "Firecrawl attempts that failed the quality gate aren't counted — " - "billed-attempt tracking needs the pipeline to persist fetch attempts." - ) - - st.divider() - st.subheader("Inspect a capture") - labels = [f"[{r['question']}] {r['url']}" for _, r in view.iterrows()] - if not labels: - st.stop() - choice = st.selectbox("URL", range(len(labels)), format_func=lambda i: labels[i]) - row = view.iloc[choice] - url = row["url"] - cap = _index(location, cfg.s3_prefix, url) - - c1, c2 = st.columns([3, 2]) - with c1: - st.markdown(f"**URL:** [{url}]({url})") - st.markdown( - f"**Question:** `{row['question']}` · **Bot:** `{row['bot']}` · " - f"**Origin:** `{row['origin'] or '—'}`" - ) - st.markdown(f"**Run:** `{row['run_id'] or '—'}`") - review = row["metaculus"] or row["question_url"] - if review: - st.markdown(f"**Metaculus question:** [{review}]({review})") - if row["comment"]: - st.markdown(f"**Cited in comment:** [{row['comment']}]({row['comment']})") - if row["query"]: - st.markdown(f"**Search query:** `{row['query']}`") - if row.get("tool_args"): - st.markdown(f"**Tool args:** `{row['tool_args']}`") - with c2: - if cap: - st.markdown( - f"**Captured:** {cap.get('captured_at','')[:19]} · " - f"**Fetcher:** {cap.get('fetcher','')} · " - f"**HTTP:** {cap.get('status_code','?')}" - ) - - if not cap: - st.warning( - "No stored capture for this URL — it failed the quality gate / errored, " - "or hasn't been captured yet." - ) - st.stop() - - tab_shot, tab_md, tab_html = st.tabs(["🖼 Screenshot", "📝 Markdown", "🌐 HTML"]) - - with tab_shot: - key = cap.get("screenshot_key") - data = _get_bytes(location, key) if key else None - if data: - st.download_button("Download .webp", data, file_name="screenshot.webp") - st.image(data, use_container_width=True) - else: - st.info("No screenshot stored.") - - with tab_md: - key = cap.get("markdown_key") - data = _get_bytes(location, key) if key else None - if data: - text = data.decode("utf-8", "replace") - st.download_button("Download .md", data, file_name="page.md") - st.caption(f"{len(text):,} chars") - st.markdown(text) - else: - st.info("No markdown stored.") - - with tab_html: - key = cap.get("html_key") - data = _get_bytes(location, key) if key else None - if data: - html = data.decode("utf-8", "replace") - st.download_button("Download .html", data, file_name="page.html") - st.caption( - f"{len(html):,} chars · rendered below (CSS/images load from the " - "original site and may not all resolve — the screenshot is the " - "faithful visual record)." - ) - st.components.v1.html(html, height=800, scrolling=True) - else: - st.info("No HTML stored.") - - -if __name__ == "__main__": - main() diff --git a/poetry.lock b/poetry.lock index f741fa95..82b06da8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -444,12 +444,11 @@ version = "2.18.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["dev"] files = [ {file = "babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35"}, {file = "babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d"}, ] -markers = {main = "extra == \"source-archive\""} [package.extras] dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] @@ -508,48 +507,6 @@ files = [ {file = "blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf"}, ] -[[package]] -name = "boto3" -version = "1.43.19" -description = "The AWS SDK for Python" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "boto3-1.43.19-py3-none-any.whl", hash = "sha256:ec6825193b75fbb6bfbf12181e4960d00ad2f404343586765394ce620e63783c"}, - {file = "boto3-1.43.19.tar.gz", hash = "sha256:8b84704719dd3960ac12a8f37d9ff5adb853715baa9742f84fdbe2de0305c4cb"}, -] - -[package.dependencies] -botocore = ">=1.43.19,<1.44.0" -jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.18.0,<0.19.0" - -[package.extras] -crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] - -[[package]] -name = "botocore" -version = "1.43.19" -description = "Low-level, data-driven core of boto 3." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "botocore-1.43.19-py3-none-any.whl", hash = "sha256:99dbdccbf748974750601e805cecc9362a85d11fee89d6d58cd3f4ff302e6ff9"}, - {file = "botocore-1.43.19.tar.gz", hash = "sha256:18ac2fdd76c89b940707eb10493ff58678adad337d03215caec2d408ccd43cc0"}, -] - -[package.dependencies] -jmespath = ">=0.7.1,<2.0.0" -python-dateutil = ">=2.1,<3.0.0" -urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3" - -[package.extras] -crt = ["awscrt (==0.32.2)"] - [[package]] name = "cachetools" version = "7.1.3" @@ -867,29 +824,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "cloakbrowser" -version = "0.3.32" -description = "Stealth Chromium that passes every bot detection test. Drop-in Playwright replacement with source-level fingerprint patches." -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "cloakbrowser-0.3.32-py3-none-any.whl", hash = "sha256:5a993ee019bfcd00d545d7d6d51837646bcb1e8226545acdf0b543b38a8883df"}, - {file = "cloakbrowser-0.3.32.tar.gz", hash = "sha256:7361e2f5e366f651b5d54aad3ac13e145462110e0956b538ae3686916c36535a"}, -] - -[package.dependencies] -httpx = ">=0.24" -playwright = ">=1.40" - -[package.extras] -dev = ["pytest (>=7.0)", "pytest-asyncio (>=0.23)"] -geoip = ["geoip2 (>=4.0)", "socksio (>=1.0)"] -patchright = ["patchright (>=1.40)"] -serve = ["aiohttp (>=3.9)", "websockets (>=12.0)"] - [[package]] name = "colorama" version = "0.4.6" @@ -1010,27 +944,6 @@ mypy = ["bokeh", "contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.17.0)", " test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] -[[package]] -name = "courlan" -version = "1.4.0" -description = "Clean, filter and sample URLs to optimize data collection – includes spam, content type and language filters." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "courlan-1.4.0-py3-none-any.whl", hash = "sha256:ad1dbdefd912ca7238d4607dc855df5df097f56bac175dd662c84eed3802f49e"}, - {file = "courlan-1.4.0.tar.gz", hash = "sha256:fbbac7b7fcde2195ea08e707609503c81cf39c891e8d26cdb1fed4585782d63d"}, -] - -[package.dependencies] -babel = ">=2.16.0" -tld = ">=0.13" -urllib3 = ">=1.26,<3" - -[package.extras] -dev = ["mypy (==2.1.0)", "pytest (==9.0.3)", "pytest-cov (==7.1.0)", "pytest-httpserver (==1.1.5)", "ruff (==0.15.15)"] - [[package]] name = "crontab" version = "1.0.5" @@ -1150,30 +1063,6 @@ typepy = {version = ">=1.3.2,<3", extras = ["datetime"]} logging = ["loguru (>=0.4.1,<1)"] test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)", "tcolorpy (>=0.1.2)"] -[[package]] -name = "dateparser" -version = "1.4.0" -description = "Date parsing library designed to parse dates from HTML pages" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "dateparser-1.4.0-py3-none-any.whl", hash = "sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378"}, - {file = "dateparser-1.4.0.tar.gz", hash = "sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4"}, -] - -[package.dependencies] -python-dateutil = ">=2.7.0" -pytz = ">=2024.2" -regex = ">=2024.9.11" -tzlocal = ">=0.2" - -[package.extras] -calendars = ["convertdate (>=2.2.1)", "hijridate"] -fasttext = ["fasttext (>=0.9.1)", "numpy (>=1.22.0,<2)"] -langdetect = ["langdetect (>=1.0.0)"] - [[package]] name = "debugpy" version = "1.8.20" @@ -1492,28 +1381,6 @@ files = [ {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, ] -[[package]] -name = "firecrawl-py" -version = "4.28.2" -description = "Python SDK for Firecrawl API" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "firecrawl_py-4.28.2-py3-none-any.whl", hash = "sha256:0689080cb01672370e5a97963e0df479f6102137aa088857eac0fa287a4269b6"}, - {file = "firecrawl_py-4.28.2.tar.gz", hash = "sha256:7e6181e2129b63c8d6aec5728d9b2fcf16ea82cb854372ad824b278efd258696"}, -] - -[package.dependencies] -aiohttp = "*" -httpx = "*" -nest-asyncio = "*" -pydantic = ">=2.0" -python-dotenv = "*" -requests = "*" -websockets = "*" - [[package]] name = "fonttools" version = "4.63.0" @@ -1813,100 +1680,6 @@ gitdb = ">=4.0.1,<5" doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] -[[package]] -name = "greenlet" -version = "3.5.1" -description = "Lightweight in-process concurrent programming" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "greenlet-3.5.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7eacb17a9d41538a2bc4912eba5ef13823c83cb69e4d141d0813debe7163187f"}, - {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e5cc9606aa5f4e0bde0d3bd502b44f743864c3ffa5cfa1011b1e30f5aa02366f"}, - {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c3d35f87c7253b715d13d679e0783d845910144f282cb939fe1ba4ac8616269c"}, - {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:00929c98ec525fd9bf075875d8c5f6a983a90906cdf78a66e6de2d8e466c2a19"}, - {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:540dae7b956209af4d70a3be35927b4055f617763771e5e84a5255bea934d2f5"}, - {file = "greenlet-3.5.1-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:001775efe7b8e758861294c7a27c28af87f3f3f1c20468a2bc618c45b346c061"}, - {file = "greenlet-3.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed8cdb691169715a9a492844a83246f090182247d1a5031dc78a403f68ba1e97"}, - {file = "greenlet-3.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d59e840387076a51016777a9328b3f2c427c6f9208a6e958bad251be50a648d"}, - {file = "greenlet-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:b9152fca4a6466e114aaec745ae61cba739903a109754a9d4e1262f01e9259b1"}, - {file = "greenlet-3.5.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:73f78f9b9f0a5c06e5c946ba1e8e36f5114923b6be109ee618c54f079c3ea14f"}, - {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0cbed8bb44e23c5b199f888f4e4ce096b45ad9f25ff74a7ad0213875e936bb2"}, - {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a203a8bd0acb0701653d3bbb26e404854a68674139ed5cbb778830f42b09bb33"}, - {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ebeb75c81211f5c702576cf81f315e77e23cfdb2c7c6fcb9dd143e6de35c360"}, - {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a271fcd66c74615cda6a964fda3f304267a12e50a084472218a39bb0376f563"}, - {file = "greenlet-3.5.1-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:017a544f0385d441e88714160d089d6900ef46c9eff9d99b6715a5ef2d127747"}, - {file = "greenlet-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ded7b068c7c31c1a8657d4fd42d886b3e051ae29f88b80c5ff9d502257b0f071"}, - {file = "greenlet-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0932b81d72f552ded9d810d00021b64d89f2195a91ce115b893f943b7a4ab3c"}, - {file = "greenlet-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:88e300d136eac057b2397aa1cfd7328b4c87c7eb66a09c7bc6a1292234db474e"}, - {file = "greenlet-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:cc6ab7e555c8a112ad3a76e368e86e12a2754bcae1652a5602e133ec7b635523"}, - {file = "greenlet-3.5.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:fa4f98af3a528f0c3fd592a26df7f376f93329c8f4d987f6bb979057af8bf5e2"}, - {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffea73584b216150eab159b6d12348fb253e68757974de1e2c40d8a318ac89ed"}, - {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1072b4f9edcc1e192d9283a66a3e68d6b84c561de33a83d7858beb9ba1effe10"}, - {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:89101bfd5011e069be974903cb3a4e4523845e4ece2d62dcd8d358933c0ef249"}, - {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:add5217d68b31130f0beca584d7fef4878327d2e31642b66618a14eef312b63b"}, - {file = "greenlet-3.5.1-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:e6cd99ea59dd5d89f0c956606571d79bfe6f68c9eb7f4a4083a41a7f1587edee"}, - {file = "greenlet-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a5ea42a752d47a145eae922b605cd1634665ac3d5ec1e72402d5048e8d60d207"}, - {file = "greenlet-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c5551170cf4f5ff5623e9af81323751979fee2c731e2287b61f73cd27257b823"}, - {file = "greenlet-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3c8bb982ad117d29478ef8f5533e97df21f1e2befd17a299257b0c96d1371c0b"}, - {file = "greenlet-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:80eb4b04dadc4e67df3fae179a32c4706a3f495bc7f22fc8a81115d5f5512188"}, - {file = "greenlet-3.5.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:51518ff74664078fc51bffcc6fc529b0df5ae58da192691cee765d45ce944a2b"}, - {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ffdb3c0bb002c99cd8f298957e046c3dbf6006b5b7cdf11a4e19194624a0a0a"}, - {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7715a5a2c3378ba602c3a440558261e13a820bb53a82693aacd7b7f6d964e283"}, - {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d40a890035c0058cadbdc4af7569800fd28a0e527a0fdbb7b5f9418f176846ce"}, - {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc71ff466927a201b08305acac451ebe1aedfcea002f62f1f2f2ac2ac1e6a135"}, - {file = "greenlet-3.5.1-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:67821bb03e4e98664490edb787ff6af501194c29bbee0f5c1dfdcf1dc3d9d436"}, - {file = "greenlet-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cd443683db272ebaaca03af98c0b063ab30db70ea8a31a1559f35e3f7b744ccd"}, - {file = "greenlet-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:089fff7a6ce8d9316d1f65ebc00273a56be258c1725b32b94de90a3a979557e1"}, - {file = "greenlet-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:110a1ca7b49b014b097f6078272c3f4ed31af45b254de5228b79adba879f6af9"}, - {file = "greenlet-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e"}, - {file = "greenlet-3.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07"}, - {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea"}, - {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2"}, - {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c09df69dc1712d131332054a858a3e5cca400967fa3a672e2324fbb0971448c"}, - {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c"}, - {file = "greenlet-3.5.1-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:a4764e0bfc6a4d114c865b32520805c16a990ef5f286a514413b05d5ecd6a23d"}, - {file = "greenlet-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0"}, - {file = "greenlet-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc"}, - {file = "greenlet-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3"}, - {file = "greenlet-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54"}, - {file = "greenlet-3.5.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad"}, - {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e"}, - {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986"}, - {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea37d5a157eb9493820d3792ac4ece28619a394391d2b9f2f78057d396ff0f0f"}, - {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e"}, - {file = "greenlet-3.5.1-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:8d8a23250ea3ec7b36de8fa4b541e9e2db3ee82915cc060ab0631609ad8b28de"}, - {file = "greenlet-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d"}, - {file = "greenlet-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78"}, - {file = "greenlet-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2"}, - {file = "greenlet-3.5.1-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2baee5ca02031757ffe8cc3d69f0cc0aec7065ce362622da74f32d3bcab1c541"}, - {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de"}, - {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64"}, - {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:10a9a1c0bfbc93d41156ffcb90c75fbc05544054faf15dcc1fdf9765f8b607f0"}, - {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5"}, - {file = "greenlet-3.5.1-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:ef08c1567c78074b22d1a200183d52d04a14df447bf70bcbb6a3507a48e776fc"}, - {file = "greenlet-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368"}, - {file = "greenlet-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26"}, - {file = "greenlet-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab"}, - {file = "greenlet-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6"}, - {file = "greenlet-3.5.1-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed"}, - {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244"}, - {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c"}, - {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a57b0d05a0448eed231d59c0ceb287dde984551e54cbc51ac2d4865712838e9c"}, - {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd"}, - {file = "greenlet-3.5.1-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:b0703c2cef53e01baec47f7a3868009913ad71ec678bbecb42a6f40895e4ce62"}, - {file = "greenlet-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e"}, - {file = "greenlet-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:17d86354f0ae6b61bf9be5148d0dd34e06c3cb7c602c671f79f29ac3b150e659"}, - {file = "greenlet-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e"}, - {file = "greenlet-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a"}, - {file = "greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829"}, -] - -[package.extras] -docs = ["Sphinx", "furo"] -test = ["objgraph", "psutil", "setuptools"] - [[package]] name = "griffelib" version = "2.0.2" @@ -1973,31 +1746,6 @@ files = [ [package.extras] tests = ["pytest"] -[[package]] -name = "htmldate" -version = "1.10.0" -description = "Fast and robust extraction of original and updated publication dates from URLs and web pages." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "htmldate-1.10.0-py3-none-any.whl", hash = "sha256:9211dae35ab94147c8ed9e5fc2c9287a5cf31d2394cb7857e7f5dd814eb2aad6"}, - {file = "htmldate-1.10.0.tar.gz", hash = "sha256:a38df10772ab5d7dbb11896e3f6a852a8491fb1b0965465bc174e23fc2baae58"}, -] - -[package.dependencies] -charset_normalizer = ">=3.4.0" -dateparser = ">=1.1.2" -lxml = ">=5.3.0" -python-dateutil = ">=2.9.0.post0" -urllib3 = ">=1.26,<3" - -[package.extras] -all = ["htmldate[dev]", "htmldate[speed]"] -dev = ["mypy", "pytest", "pytest-cov", "ruff", "types-dateparser", "types-lxml", "types-python-dateutil", "types-urllib3"] -speed = ["backports-datetime-fromisoformat ; python_version < \"3.11\"", "faust-cchardet (>=2.1.19)", "urllib3[brotli]"] - [[package]] name = "httpcore" version = "1.0.9" @@ -2522,19 +2270,6 @@ files = [ {file = "jiter-0.15.0.tar.gz", hash = "sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76"}, ] -[[package]] -name = "jmespath" -version = "1.1.0" -description = "JSON Matching Expressions" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, - {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, -] - [[package]] name = "joblib" version = "1.5.3" @@ -2876,22 +2611,6 @@ docs = ["autodoc-traits", "jinja2 (<3.2.0)", "mistune (<4)", "myst-parser", "pyd openapi = ["openapi-core (>=0.18.0,<0.19.0)", "ruamel-yaml"] test = ["hatch", "ipykernel", "openapi-core (>=0.18.0,<0.19.0)", "openapi-spec-validator (>=0.6.0,<0.8.0)", "pytest (>=7.0,<8)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter[server] (>=0.6.2)", "pytest-timeout", "requests-mock", "ruamel-yaml", "sphinxcontrib-spelling", "strict-rfc3339", "werkzeug"] -[[package]] -name = "justext" -version = "3.0.2" -description = "Heuristic based boilerplate removal tool" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7"}, - {file = "justext-3.0.2.tar.gz", hash = "sha256:13496a450c44c4cd5b5a75a5efcd9996066d2a189794ea99a49949685a0beb05"}, -] - -[package.dependencies] -lxml = {version = ">=4.4.2", extras = ["html-clean"]} - [[package]] name = "kiwisolver" version = "1.5.0" @@ -3114,176 +2833,6 @@ semantic-router = ["aurelio-sdk (==0.0.19) ; python_full_version < \"3.14.0\"", stt-nvidia-riva = ["audioread (>=3.0.1)", "numpy (>=1.26.0)", "nvidia-riva-client (>=2.15.0)", "soundfile (>=0.12.1)"] utils = ["numpydoc (==1.8.0)"] -[[package]] -name = "lxml" -version = "6.1.1" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "lxml-6.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:09dd5b7075dc2f7709654a46543ba1ea3c2e217b2ed8fbd413a8a945a0f40f60"}, - {file = "lxml-6.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f6ac4ef4d82dff54670227a69c67782ae0b811b5cf6b17954f1e8f7502fc0d1d"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:556e94a63c9b04716f8e4de2abb65775061f846e89331b6c5be79183a24f98ea"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6bf403fbb3b3e348a561a5f4f0b9961835657981c802a1df03653eef8a9074"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dde6131244bba38a17c745836ba190bc753fd73c9291666287fd0a3fa3dcf30"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98fc784c2c1440667aeedf8465bdfe10208acf0ead656a2c68627299f546b315"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:add8cf6ddf9a65116119a28ece0f7886e30af27ba724a7594305f1d1b58a92a1"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:cf9d57306d848218f3601fee7601fab1a327c942d56e2e97610583cb4dd74206"}, - {file = "lxml-6.1.1-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88136950da4d13c318bde414ce10219931937851327f44328f2df4d2c4614067"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cecdd5dfdc87b1fd87dbf81d4b037a544f47f4c744200a67013771682d67686a"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cd312b9692e831d2ffcad61eab31d91d4b4655a962e61de8fb410472cbcd37aa"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:5b7328b46d49fc9477d91ae8f6d55340347d827b7734ba3ea33faae0efef1383"}, - {file = "lxml-6.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37a58976370f36d9329d118ad0b953c5aeb9119ac9c6a4e258942a225d0573a1"}, - {file = "lxml-6.1.1-cp310-cp310-win32.whl", hash = "sha256:cea3f4c1af79af13cdb2da0c028111d8f8522d4f22a000c82385535f24e5cf3a"}, - {file = "lxml-6.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3abf332af33a74288675d936fe861fd4344da0dd6622193fbc4f2bfbb35536b5"}, - {file = "lxml-6.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:8dadbe5b217ff35b6a8d16610dd710219b59b76d13f0e3f0d9f36786206e4485"}, - {file = "lxml-6.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:53b7d2b7a10b1c35c0a5e21e9224accf60c1bbfba523990732e521b2b73adef2"}, - {file = "lxml-6.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ff3f333630ab480244a1bff72043e511a91eb22e7595dead8653ee5612dd8f3d"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a4bbea04c97f6d78a48e3fbc1cb9116d2780b1b39e03a23f6eb9b603fd61f510"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db1d75f6617a49c1c01bc7023713e0ff59ab32c9579ae62a7674c0e34f3b0b0a"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a12689be69a28ddaa0ab99a5a1137da2afd5f8f16df7b5680b66f616d3eda1d"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b73c339ae29b90fd2d06e58ebd555a751bde9cd6bbd36cc0281b9a2c94e9d8"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:752d3bbfe874715ccd0aec7f88d7fc623c0f1fd7aa7b3238a084e017bad2a009"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:6b1761fbf9ec984e2e9d9c589ef5f5fd684b7c19f92aadd567a26c5224958db6"}, - {file = "lxml-6.1.1-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d680fbcb768404c601ecb43519ecd8461f6954cb11c06a78962f666832ccfca8"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:162af1091cd785f2f27e62d3547ae9bc58ec5c86dd314d67021fd02463708d83"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e9308ff8241c532df3f3e570f9a5aeed6c853f888512ba4b75638d7c11c95ef6"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5f6994074ebae6ffb04447268e37dc16edc304f9859cf91acb86e0af6c1b395c"}, - {file = "lxml-6.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80c2dfadb855da477cf73373ad29a333535dedb9b12bad02c9814c8e2b43bf08"}, - {file = "lxml-6.1.1-cp311-cp311-win32.whl", hash = "sha256:30a89d3ac8faec007453fb541f3f46807eeec88edd5826f6e3fe001752a2c621"}, - {file = "lxml-6.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:abbefa31eee84842140f67acef1c828e28bba8bbf0c3bc6e5492a9af88152c28"}, - {file = "lxml-6.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:dcb292aa7fe485ceff7af4f92e46c5af397daec5dff64871a528f0fc47a3cc5b"}, - {file = "lxml-6.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7"}, - {file = "lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f"}, - {file = "lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955"}, - {file = "lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a"}, - {file = "lxml-6.1.1-cp312-cp312-win32.whl", hash = "sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a"}, - {file = "lxml-6.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77"}, - {file = "lxml-6.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f"}, - {file = "lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736"}, - {file = "lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f"}, - {file = "lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785"}, - {file = "lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947"}, - {file = "lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca"}, - {file = "lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660"}, - {file = "lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc"}, - {file = "lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0"}, - {file = "lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245"}, - {file = "lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590"}, - {file = "lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb"}, - {file = "lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603"}, - {file = "lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137"}, - {file = "lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf"}, - {file = "lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee"}, - {file = "lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038"}, - {file = "lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2"}, - {file = "lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e"}, - {file = "lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1"}, - {file = "lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e"}, - {file = "lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c"}, - {file = "lxml-6.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6689e828a94eee4f139408c337bb198e014724bb8a8c26d3cfac49d119ed69a6"}, - {file = "lxml-6.1.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bdebcc8a75d38c7598dfb2c9ed852d7a9eb4a10d6e2d0764b919b802bf32ac88"}, - {file = "lxml-6.1.1-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8be8ad51249698103d24b0571df35a10990fbe93dd043b6c024172189485f5e3"}, - {file = "lxml-6.1.1-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:76447f65250ed2501ead1a1552f5ce8edff159a86f308348e6a9c4acb5e1f1b4"}, - {file = "lxml-6.1.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ffecec8eb889b58ba9be5b95fb1cc78e22ea8eedea38e8736a1568fe1979250e"}, - {file = "lxml-6.1.1-cp38-cp38-win32.whl", hash = "sha256:c674693f055fa2495de12292cb45e9944199d8eaef5a2dec45175c7c61cb73e3"}, - {file = "lxml-6.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:55b03549819867ea141c0202242c4816c82e52ec36e7e648db9d8da5a3dc3ed6"}, - {file = "lxml-6.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c9f79d5325907f13e1be0b3e4dacc1049d1dffc4aeee3c995284bea5fe0fab7d"}, - {file = "lxml-6.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:83b6b30eb131da7a75b601f28c5d6971e6ed3e887919bf6b6a1ad3c2df289080"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:441dd227fa0690eb9fc81edabc63cdcefc212bba99b906dcf6e32cc1a9d3e533"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e07c65f443c887bbcf31cc1771d932ecc192a5273943589b3c7572b749f1ffb2"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5bec7d03d78d853597d6107854c2310ce3f761fd218fe9fe91d5101fcf6c2efe"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f76acfb5f68ba982635a53fd985a8044be98a35b43232c2a1ee235ffab3e1dd"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:8d43ca737b20e106e4aebc42b2f3ae19f00ba63d7eb731698ee083d72d15646f"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:32ab449a5486f6c758e849bb86710d0e45edc24a04e250c01555f8f5653958f8"}, - {file = "lxml-6.1.1-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53c909b62a0532183542fed00c5a7218258c56292d409bc789886fe1cb04c438"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:640f97d43d867bcb9c75b3af013b64850756b746cb6bce8ace83b70da3abba9d"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:469e3618338bd7ab5beb412d2439825479fcf0dab99e394ca563dbc4eaf6c834"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:aae97dfdb60715c164419ac2532a76d013c3918a665eb6cb7288098b5f349aaf"}, - {file = "lxml-6.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c9a4b821dc7055bf9e05ff5719e18ec501f75c0f0bbfabd573b277559780833d"}, - {file = "lxml-6.1.1-cp39-cp39-win32.whl", hash = "sha256:639f6c857d91d9be29bd7502348d6736dab168b54b5158cd899abf11684dc186"}, - {file = "lxml-6.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:34c2d737beabfe35baada43941ed519251e9a12e779031496bcd5d539fcfd730"}, - {file = "lxml-6.1.1-cp39-cp39-win_arm64.whl", hash = "sha256:07a4a68e286ee7a1ed7dfb8af83e615757c0ccfe9f18c6b4ea6771388d9ba8c9"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:31033dc34636ea6b7d5cc11b1ddbda78a14de858ba9d3e1ed4b69a3085bc521e"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3893c14c4b6ac5b2d54ba8cf03e99fe5104e592de491f19bd6b82756c09f8004"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c07da4cebf6889f03ebac8d238f62318e29f495de0aa18a51ea14e61ae907e2e"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f0ce10945fab9c4c06ce14e22af9059d1a87493a9af4501a5b0b9187e21cf2"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8844cd288697c6425c9beba919302241e3278871dc6519515e72b04e987abcf"}, - {file = "lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84"}, - {file = "lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40"}, -] - -[package.dependencies] -lxml_html_clean = {version = "*", optional = true, markers = "extra == \"html-clean\""} - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml_html_clean"] -html5 = ["html5lib"] -htmlsoup = ["BeautifulSoup4"] - -[[package]] -name = "lxml-html-clean" -version = "0.4.5" -description = "HTML cleaner from lxml project" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "lxml_html_clean-0.4.5-py3-none-any.whl", hash = "sha256:c76fcadd1e5bfb9b8bafc2200d51e4e78eb0dad67f56881c21dfb6484c7e7746"}, - {file = "lxml_html_clean-0.4.5.tar.gz", hash = "sha256:e2a4c7d5beedd17cd7b484d848a0571e54baa239a4f9df5546e3acba7f990560"}, -] - -[package.dependencies] -lxml = ">=6.1.1" - [[package]] name = "markdown-it-py" version = "4.2.0" @@ -4561,29 +4110,6 @@ files = [ {file = "platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a"}, ] -[[package]] -name = "playwright" -version = "1.60.0" -description = "A high-level API to automate web browsers" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7"}, - {file = "playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5"}, - {file = "playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705"}, - {file = "playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e"}, - {file = "playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353"}, - {file = "playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7"}, - {file = "playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02"}, - {file = "playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537"}, -] - -[package.dependencies] -greenlet = ">=3.1.1,<4.0.0" -pyee = ">=13,<14" - [[package]] name = "plotly" version = "6.7.0" @@ -5339,25 +4865,6 @@ numpy = ">=1.16.4" carto = ["pydeck-carto"] jupyter = ["ipykernel (>=5.1.2)", "ipywidgets (>=7,<8)", "traitlets (>=4.3.2)"] -[[package]] -name = "pyee" -version = "13.0.1" -description = "A rough port of Node.js's EventEmitter to Python with a few tricks of its own" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228"}, - {file = "pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8"}, -] - -[package.dependencies] -typing-extensions = "*" - -[package.extras] -dev = ["black", "build", "flake8", "flake8-black", "isort", "jupyter-console", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocstrings[python]", "mypy", "pytest", "pytest-asyncio ; python_version >= \"3.4\"", "pytest-trio ; python_version >= \"3.7\"", "sphinx", "toml", "tox", "trio", "trio ; python_version > \"3.6\"", "trio-typing ; python_version > \"3.6\"", "twine", "twisted", "validate-pyproject[all]"] - [[package]] name = "pygments" version = "2.20.0" @@ -5394,46 +4901,6 @@ dev = ["coverage[toml] (==7.10.7)", "cryptography (>=3.4.0)", "pre-commit", "pyt docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==7.10.7)", "pytest (>=8.4.2,<9.0.0)"] -[[package]] -name = "pymupdf" -version = "1.27.2.3" -description = "A high performance Python library for data extraction, analysis, conversion & manipulation of PDF (and other) documents." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "pymupdf-1.27.2.3-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc1bc3cae6e9e150b0dbb0a9221bdfd411d65f0db2fe359eaa22467d7cc2a05f"}, - {file = "pymupdf-1.27.2.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:660d93cb6da5bbddf11d3982ae27745dd3a9902d9f24cdb69adab83962294b5a"}, - {file = "pymupdf-1.27.2.3-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1dd460a3ae4597a755f00a3bd9771f5ebf1531dc111f6a36bf05dd00a6b84425"}, - {file = "pymupdf-1.27.2.3-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:857842b4888827bd6155a1131341b2822a7ebe9a8c15a975fd7d490d7a64a30c"}, - {file = "pymupdf-1.27.2.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:580983849c64a08d08344ca3d1580e87c01f046a8392421797bc850efd72a5b6"}, - {file = "pymupdf-1.27.2.3-cp310-abi3-win32.whl", hash = "sha256:a5c1088a87189891a4946ab314a14b7934ac4c5b6077f7e74ebee956f8906d0e"}, - {file = "pymupdf-1.27.2.3-cp310-abi3-win_amd64.whl", hash = "sha256:d20f68ef15195e073071dbc4ae7455257c7889af7584e39df490c0a92728526e"}, - {file = "pymupdf-1.27.2.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:77691604c5d1d0233827139bbcdea61fd57879c84712b8e49b1f45520f7ab9c2"}, - {file = "pymupdf-1.27.2.3.tar.gz", hash = "sha256:7a92faa25129e8bbec5e50eeb9214f187665428c31b05c4ef6e36c58c0b1c6d2"}, -] - -[[package]] -name = "pymupdf4llm" -version = "0.3.4" -description = "PyMuPDF Utilities for LLM/RAG" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "pymupdf4llm-0.3.4-py3-none-any.whl", hash = "sha256:0517492f82af978541162ade20fc54649cdca52acd478e33b97cb6171d69956f"}, - {file = "pymupdf4llm-0.3.4.tar.gz", hash = "sha256:48d396a5fb3c14351493c7f1dd25b2a843efdbdc4526e489ee100643a2cebec1"}, -] - -[package.dependencies] -pymupdf = ">=1.27.1" -tabulate = "*" - -[package.extras] -layout = ["pymupdf-layout (>=1.27.1)"] - [[package]] name = "pyparsing" version = "3.3.2" @@ -5730,12 +5197,11 @@ version = "2026.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" -groups = ["main", "dev"] +groups = ["dev"] files = [ {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, ] -markers = {main = "extra == \"source-archive\""} [[package]] name = "pywin32" @@ -6336,25 +5802,6 @@ files = [ {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, ] -[[package]] -name = "s3transfer" -version = "0.18.0" -description = "An Amazon S3 Transfer Manager" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "s3transfer-0.18.0-py3-none-any.whl", hash = "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6"}, - {file = "s3transfer-0.18.0.tar.gz", hash = "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd"}, -] - -[package.dependencies] -botocore = ">=1.37.4,<2.0a0" - -[package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] - [[package]] name = "scikit-learn" version = "1.8.0" @@ -6722,22 +6169,6 @@ typepy = ">=1.2.0,<3" logging = ["loguru (>=0.4.1,<1)"] test = ["pytablewriter (>=0.46)", "pytest"] -[[package]] -name = "tabulate" -version = "0.10.0" -description = "Pretty-print tabular data" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3"}, - {file = "tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d"}, -] - -[package.extras] -widechars = ["wcwidth"] - [[package]] name = "tcolorpy" version = "0.1.7" @@ -7091,27 +6522,6 @@ webencodings = ">=0.4" doc = ["sphinx", "sphinx_rtd_theme"] test = ["pytest", "ruff"] -[[package]] -name = "tld" -version = "0.13.2" -description = "Extract the top-level domain (TLD) from the URL given." -optional = true -python-versions = ">=3.7" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c"}, - {file = "tld-0.13.2.tar.gz", hash = "sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345"}, -] - -[package.extras] -all = ["tld[build,dev,docs,lint,test]"] -build = ["build", "pkginfo", "twine", "wheel"] -dev = ["detect-secrets", "ipython", "uv"] -docs = ["sphinx", "sphinx-autobuild", "sphinx-llms-txt-link", "sphinx-no-pragma", "sphinx-rtd-theme (>=1.3.0)", "sphinx-source-tree ; python_version > \"3.9\""] -lint = ["doc8", "mypy", "pydoclint", "ruff"] -test = ["coverage", "fake.py", "pytest", "pytest-codeblock", "pytest-cov", "pytest-ordering", "tox"] - [[package]] name = "tokenizers" version = "0.22.2" @@ -7285,32 +6695,6 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] -[[package]] -name = "trafilatura" -version = "2.0.0" -description = "Python & Command-line tool to gather text and metadata on the Web: Crawling, scraping, extraction, output as CSV, JSON, HTML, MD, TXT, XML." -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "trafilatura-2.0.0-py3-none-any.whl", hash = "sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d"}, - {file = "trafilatura-2.0.0.tar.gz", hash = "sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247"}, -] - -[package.dependencies] -certifi = "*" -charset_normalizer = ">=3.4.0" -courlan = ">=1.3.2" -htmldate = ">=1.9.2" -justext = ">=3.0.1" -lxml = {version = ">=5.3.0", markers = "platform_system != \"Darwin\" or python_version > \"3.8\""} -urllib3 = ">=1.26,<3" - -[package.extras] -all = ["brotli", "cchardet (>=2.1.7) ; python_version < \"3.11\"", "faust-cchardet (>=2.1.19) ; python_version >= \"3.11\"", "htmldate[speed] (>=1.9.2)", "py3langid (>=0.3.0)", "pycurl (>=7.45.3)", "urllib3[socks]", "zstandard (>=0.23.0)"] -dev = ["flake8", "mypy", "pytest", "pytest-cov", "types-lxml", "types-urllib3"] - [[package]] name = "traitlets" version = "5.15.0" @@ -7456,25 +6840,6 @@ files = [ {file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"}, ] -[[package]] -name = "tzlocal" -version = "5.3.1" -description = "tzinfo object for the local timezone" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"source-archive\"" -files = [ - {file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"}, - {file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"}, -] - -[package.dependencies] -tzdata = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"] - [[package]] name = "unidecode" version = "1.4.0" @@ -7896,10 +7261,7 @@ enabler = ["pytest-enabler (>=3.4)"] test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] -[extras] -source-archive = ["boto3", "cloakbrowser", "firecrawl-py", "playwright", "pymupdf4llm", "trafilatura"] - [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "d9abd6c9194bdd4769704c8c60f48f438f9d77370b35ee739555d3b9fd3e5e22" +content-hash = "4cf8a2f0d78535d469e1c0c647146d2f890f94f66c6f37fe7128376b958f6d46" diff --git a/pyproject.toml b/pyproject.toml index c8b322f6..705eda4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,26 +51,6 @@ hyperbrowser = ">=0.53.0,<1.0.0" pendulum = "^3.1.0" openai-agents = {extras = ["litellm"], version = ">=0.2.0,<0.20.0"} -# Optional backends for the source archive (agents_and_tools/source_archive). -# Install with: pip install forecasting-tools[source-archive] -boto3 = {version = ">=1.34,<2.0.0", optional = true} -playwright = {version = ">=1.44,<2.0.0", optional = true} -firecrawl-py = {version = ">=4.0,<5.0.0", optional = true} -trafilatura = {version = ">=1.9,<3.0.0", optional = true} -pymupdf4llm = {version = ">=0.0.17,<1.0.0", optional = true} -# Self-hosted anti-bot backend (CloakBrowser). Pinned tight to 0.3.x: it's a -# young, fast-moving 0.x package whose launch() API changed recently, so bump -# the minor deliberately. The pip wheel is light (httpx + playwright); the -# ~200MB patched Chromium downloads at first launch, not at install. -cloakbrowser = {version = ">=0.3.31,<0.4.0", optional = true} - -[tool.poetry.extras] -# hyperbrowser is already a core dep (used elsewhere too). -source-archive = ["boto3", "playwright", "firecrawl-py", "trafilatura", "pymupdf4llm", "cloakbrowser"] - -[tool.poetry.scripts] -source-archive = "forecasting_tools.agents_and_tools.source_archive.cli:main" - [tool.poetry.group.dev.dependencies] time-machine = ">=2.19.0,<4.0.0" pre-commit = "^4.0.1"