diff --git a/.github/workflows/nightly-bridge.yml b/.github/workflows/nightly-bridge.yml index aeaa7edf..6a3e2f42 100644 --- a/.github/workflows/nightly-bridge.yml +++ b/.github/workflows/nightly-bridge.yml @@ -60,6 +60,8 @@ jobs: pip install ruff black - name: Run Integration Tests + # Specifically verifies wdr CLI's ability to output robust Markdown + # with accurate LaTeX and code blocks on static/JS-heavy sites run: PYTHONPATH=. pytest tests/integration/ env: EXA_API_KEY: ${{ secrets.EXA_API_KEY }} diff --git a/tests/integration/test_nightly_bridge.py b/tests/integration/test_nightly_bridge.py index 00ec628f..b009d7c8 100644 --- a/tests/integration/test_nightly_bridge.py +++ b/tests/integration/test_nightly_bridge.py @@ -90,3 +90,22 @@ def test_nightly_llm_ready_standards(): assert "[ANCHOR: TECHNICAL_DETAILS]" in content assert "[ANCHOR: COMPARISON]" in content assert "[ANCHOR: CITATIONS]" in content + + +@pytest.mark.integration +def test_nightly_js_heavy_site_parsing(): + """Verify that JS-heavy sites have their content correctly parsed without unparsed code blocks.""" + url = "https://react.dev/learn" + + result = subprocess.run( + [CLI_PATH, "resolve", url, "--provider", "jina"], + capture_output=True, + text=True, + check=True, + ) + + content = result.stdout + assert len(content) > 500, "Content too short" + assert "React" in content + assert " tag" + assert "" not in content, "Found unparsed tag" diff --git a/tests/test_content_clean.py b/tests/test_content_clean.py index 5d3fa420..0272cd9e 100644 --- a/tests/test_content_clean.py +++ b/tests/test_content_clean.py @@ -8,6 +8,9 @@

API Reference

The resolve_url function accepts a URL and returns resolved content.

It supports multiple providers including jina, firecrawl, and direct fetch.

+

To ensure high quality, the system filters out boilerplate, parses complex structures, + deduplicates headers, preserves nested lists, handles various character encodings, and + ensures the output matches 2026 LLM-ready standards with zero slop and maximum density.

diff --git a/web/package-lock.json b/web/package-lock.json index 4eae1b69..8a57e493 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -34,7 +34,7 @@ "eslint-plugin-playwright": "^2.10.5", "eslint-plugin-react-hooks": "^7.1.1", "globals": "^17.7.0", - "postcss": "8.5.19", + "postcss": "8.5.25", "tailwindcss": "^4.0.0", "typescript": "6.0.3", "vitest": "^4.1.10", @@ -3803,9 +3803,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { @@ -7019,9 +7019,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.17", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", + "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", "funding": [ { "type": "github", @@ -7427,9 +7427,9 @@ } }, "node_modules/postcss": { - "version": "8.5.19", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", - "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "funding": [ { "type": "opencollective", @@ -7446,7 +7446,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/web/package.json b/web/package.json index 59f80185..9371d8df 100644 --- a/web/package.json +++ b/web/package.json @@ -43,7 +43,7 @@ "eslint-plugin-playwright": "^2.10.5", "eslint-plugin-react-hooks": "^7.1.1", "globals": "^17.7.0", - "postcss": "8.5.19", + "postcss": "8.5.25", "tailwindcss": "^4.0.0", "typescript": "6.0.3", "vitest": "^4.1.10", @@ -52,10 +52,10 @@ "overrides": { "@ungap/structured-clone": "1.3.1", "sharp": "0.35.3", - "brace-expansion": ">=1.1.16", + "brace-expansion": "^1.1.18 || ^2.1.4 || ^3.0.6 || ^5.0.9", "axios": ">=1.18.0", "next": { - "postcss": "8.5.19" + "postcss": "8.5.25" } } }