diff --git a/.github/workflows/nightly-bridge.yml b/.github/workflows/nightly-bridge.yml index aeaa7edf..6a3e2f42 100644 --- a/.github/workflows/nightly-bridge.yml +++ b/.github/workflows/nightly-bridge.yml @@ -60,6 +60,8 @@ jobs: pip install ruff black - name: Run Integration Tests + # Specifically verifies wdr CLI's ability to output robust Markdown + # with accurate LaTeX and code blocks on static/JS-heavy sites run: PYTHONPATH=. pytest tests/integration/ env: EXA_API_KEY: ${{ secrets.EXA_API_KEY }} diff --git a/tests/integration/test_nightly_bridge.py b/tests/integration/test_nightly_bridge.py index 00ec628f..b009d7c8 100644 --- a/tests/integration/test_nightly_bridge.py +++ b/tests/integration/test_nightly_bridge.py @@ -90,3 +90,22 @@ def test_nightly_llm_ready_standards(): assert "[ANCHOR: TECHNICAL_DETAILS]" in content assert "[ANCHOR: COMPARISON]" in content assert "[ANCHOR: CITATIONS]" in content + + +@pytest.mark.integration +def test_nightly_js_heavy_site_parsing(): + """Verify that JS-heavy sites have their content correctly parsed without unparsed code blocks.""" + url = "https://react.dev/learn" + + result = subprocess.run( + [CLI_PATH, "resolve", url, "--provider", "jina"], + capture_output=True, + text=True, + check=True, + ) + + content = result.stdout + assert len(content) > 500, "Content too short" + assert "React" in content + assert "
tag"
+ assert "" not in content, "Found unparsed tag"
diff --git a/tests/test_content_clean.py b/tests/test_content_clean.py
index 5d3fa420..0272cd9e 100644
--- a/tests/test_content_clean.py
+++ b/tests/test_content_clean.py
@@ -8,6 +8,9 @@
API Reference
The resolve_url function accepts a URL and returns resolved content.
It supports multiple providers including jina, firecrawl, and direct fetch.
+ To ensure high quality, the system filters out boilerplate, parses complex structures,
+ deduplicates headers, preserves nested lists, handles various character encodings, and
+ ensures the output matches 2026 LLM-ready standards with zero slop and maximum density.