docs(agents): align synthesis prompts with August 2026 standards

.agents/skills/do-web-doc-resolver/scripts/cache_negative.py

@@ -2,8 +2,11 @@
 Negative caching logic for the Web Doc Resolver.
 """
 
+import logging
 from datetime import datetime, timedelta, timezone
 
+logger = logging.getLogger(__name__)
+
 
 def should_skip_from_negative_cache(cache, key: str, provider: str) -> bool:
     if cache is None:
@@ -24,6 +27,7 @@ def should_skip_from_negative_cache(cache, key: str, provider: str) -> bool:
             dt = dt.replace(tzinfo=timezone.utc)
         return dt > datetime.now(timezone.utc)
     except Exception:
+        logger.debug("Failed to parse negative cache expiry: %s", expires_at, exc_info=True)
         return False
 
 
@@ -49,3 +53,19 @@ def write_negative_cache(
         "metadata": metadata,
     }
     cache.set(f"neg:{provider}:{key}", entry, expire=ttl_seconds)
+
+
+def should_skip_from_bot_challenge_cache(
+    provider: str,
+    url: str,
+    bot_challenge_cache: dict[str, set[str]],
+) -> bool:
+    """Skip plain-fetch providers for URLs known to serve bot challenges."""
+    from urllib.parse import urlparse
+
+    try:
+        domain = urlparse(url).netloc
+        return provider in ("direct_fetch",) and domain in bot_challenge_cache.get(provider, set())
+    except Exception as e:
+        logger.debug("Bot challenge cache lookup failed for %s: %s", url, e)
+        return False

.agents/skills/do-web-doc-resolver/scripts/constants.py

@@ -5,8 +5,13 @@
 import os
 import typing
 
+from scripts.models import FetchTier
+
 logger = logging.getLogger(__name__)
 
+if typing.TYPE_CHECKING:
+    pass
+
 
 def _load_config() -> dict[str, typing.Any]:
     config_path = os.getenv("DO_WDR_CONFIG") or "config.toml"
@@ -110,3 +115,16 @@ def _env(
 BLOCKED_SCHEMES: set[str] = {"file", "javascript", "data", "vbscript"}
 
 DNS_CACHE_TTL: int = 60
+
+CLEAN_CONTENT: bool = os.environ.get("WDR_CLEAN_CONTENT", "1") != "0"
+
+PROVIDER_TIERS: dict[str, FetchTier] = {
+    "llms_txt": FetchTier.FREE_STATIC,
+    "direct_fetch": FetchTier.FREE_DIRECT,
+    "duckduckgo": FetchTier.FREE_SEARCH,
+    "jina": FetchTier.PAID_LITE,
+    "firecrawl": FetchTier.PAID_LITE,
+    "visual_clip": FetchTier.PAID_LITE,
+    "stealth": FetchTier.STEALTH,
+    "mistral_browser": FetchTier.PAID_BROWSER,
+}

.agents/skills/do-web-doc-resolver/scripts/models.py

@@ -20,6 +20,7 @@ class ErrorType(Enum):
     INVALID_RESPONSE = "invalid_response"
     SSRF_BLOCKED = "ssrf_blocked"
     CONTENT_TOO_LARGE = "content_too_large"
+    BOT_CHALLENGE = "bot_challenge"
     UNKNOWN = "unknown"
 
 
@@ -50,6 +51,18 @@ def max_hops(self) -> int:
         return 4
 
 
+class FetchTier(int, Enum):
+    """Escalation cost tier for fetch providers.
+    Lower = cheaper, always tried first."""
+
+    FREE_STATIC = 0  # llms_txt: static text file, zero cost
+    FREE_DIRECT = 1  # direct_fetch: plain httpx, zero cost
+    FREE_SEARCH = 2  # duckduckgo: free web search
+    PAID_LITE = 3  # jina, firecrawl: paid but cheap per-call
+    STEALTH = 4  # anti-bot bypass tier
+    PAID_BROWSER = 5  # mistral_browser: paid + JS execution
+
+
 class ProviderType(Enum):
     """Available providers for resolution."""
 
@@ -71,6 +84,7 @@ class ProviderType(Enum):
     # New providers
     DOCLING = "docling"
     OCR = "ocr"
+    VISUAL_CLIP = "visual_clip"
 
     def is_paid(self) -> bool:
         return self in (
@@ -80,6 +94,7 @@ def is_paid(self) -> bool:
             ProviderType.FIRECRAWL,
             ProviderType.MISTRAL_WEBSEARCH,
             ProviderType.MISTRAL_BROWSER,
+            ProviderType.VISUAL_CLIP,
         )
 
     def is_fast(self) -> bool:
@@ -179,3 +194,15 @@ class ReadonlyResolverProtocol(Protocol):
     """
 
     def __call__(self) -> ResolvedResult | str | None: ...
+
+
+__all__ = [
+    "ErrorType",
+    "Profile",
+    "ProviderType",
+    "ValidationResult",
+    "ProviderMetric",
+    "ResolveMetrics",
+    "ResolvedResult",
+    "ReadonlyResolverProtocol",
+]