diff --git a/.agents/skills/do-web-doc-resolver/scripts/quality.py b/.agents/skills/do-web-doc-resolver/scripts/quality.py index de6fc171..b7150095 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/quality.py +++ b/.agents/skills/do-web-doc-resolver/scripts/quality.py @@ -86,8 +86,12 @@ def _check_jargon(text_lower: str) -> bool: "transform", "supercharge", ] - jargon_count = sum(text_lower.count(signal) for signal in jargon_signals) - return jargon_count > THRESHOLD_JARGON + jargon_count = 0 + for signal in jargon_signals: + jargon_count += text_lower.count(signal) + if jargon_count > THRESHOLD_JARGON: + return True + return False def _check_frontmatter(text: str) -> bool: @@ -115,8 +119,12 @@ def _check_anchors(text: str) -> bool: def _compute_noise(text_lower: str) -> bool: """Detect noisy signals like cookie/subscribe prompts.""" noisy_signals = ["cookie", "subscribe", "javascript", "log in", "sign up"] - noise_count = sum(text_lower.count(signal) for signal in noisy_signals) - return noise_count > THRESHOLD_NOISE + noise_count = 0 + for signal in noisy_signals: + noise_count += text_lower.count(signal) + if noise_count > THRESHOLD_NOISE: + return True + return False def is_bot_challenge(content: str) -> bool: diff --git a/scripts/quality.py b/scripts/quality.py index 9ee6b6ff..ff8db32f 100644 --- a/scripts/quality.py +++ b/scripts/quality.py @@ -86,8 +86,12 @@ def _check_jargon(text_lower: str) -> bool: "transform", "supercharge", ] - jargon_count = sum(text_lower.count(signal) for signal in jargon_signals) - return jargon_count > THRESHOLD_JARGON + jargon_count = 0 + for signal in jargon_signals: + jargon_count += text_lower.count(signal) + if jargon_count > THRESHOLD_JARGON: + return True + return False def _check_frontmatter(text: str) -> bool: @@ -115,8 +119,12 @@ def _check_anchors(text: str) -> bool: def _compute_noise(text_lower: str) -> bool: """Detect noisy signals like cookie/subscribe prompts.""" noisy_signals = ["cookie", "subscribe", "javascript", "log in", "sign up"] - noise_count = sum(text_lower.count(signal) for signal in noisy_signals) - return noise_count > THRESHOLD_NOISE + noise_count = 0 + for signal in noisy_signals: + noise_count += text_lower.count(signal) + if noise_count > THRESHOLD_NOISE: + return True + return False def is_bot_challenge(content: str) -> bool: diff --git a/tests/test_content_clean.py b/tests/test_content_clean.py index 5d3fa420..981a500f 100644 --- a/tests/test_content_clean.py +++ b/tests/test_content_clean.py @@ -8,6 +8,7 @@

API Reference

The resolve_url function accepts a URL and returns resolved content.

It supports multiple providers including jina, firecrawl, and direct fetch.

+

This is additional description text designed to make the main content segment much longer than two hundred characters. This is needed so that the content cleaning utility does not fall back to raw HTML tag stripping, which would keep footer items like the secondary notices.