diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py index e9476409..e45405c0 100644 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py +++ b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py @@ -70,3 +70,13 @@ def test_distinct_pages_keep_distinct_hashes(): def test_empty_and_none_safe(): assert canonicalize_url("") == "" + + +def test_lazy_port_valueerror_returns_raw() -> None: + # .port raises ValueError lazily when the port section is non-numeric — + # junk the URL regex can extract from CSS-ish text ("Port could not be + # cast to integer value as 'root{--novem-render-frac'"). canonicalize + # must not propagate it. + junk = "http://a.test:root{--novem-render-frac/x" + assert canonicalize_url(junk) == junk + assert canonicalize_url(canonicalize_url(junk)) == canonicalize_url(junk) diff --git a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py index b791a47e..7d722c69 100644 --- a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py +++ b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py @@ -73,27 +73,31 @@ def canonicalize_url(url: str) -> str: if not url: return url raw = url.strip() + # urlsplit() itself rarely raises; .hostname/.port are LAZY properties that + # raise ValueError on junk like "http://root{--x:80/" or bad IPv6 — so the + # guard must cover the whole netloc normalization, not just the split. try: parts = urlsplit(raw) + if parts.scheme not in ("http", "https") or not parts.netloc: + return raw + + scheme = parts.scheme.lower() + + # netloc: lowercase host (bracket IPv6), keep userinfo, strip default + # port. + host = (parts.hostname or "").lower() + if ":" in host: # IPv6 literal + host = f"[{host}]" + netloc = host + if parts.username is not None: + auth = parts.username + if parts.password is not None: + auth += f":{parts.password}" + netloc = f"{auth}@{netloc}" + if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get(scheme): + netloc += f":{parts.port}" except ValueError: return raw - if parts.scheme not in ("http", "https") or not parts.netloc: - return raw - - scheme = parts.scheme.lower() - - # netloc: lowercase host (bracket IPv6), keep userinfo, strip default port. - host = (parts.hostname or "").lower() - if ":" in host: # IPv6 literal - host = f"[{host}]" - netloc = host - if parts.username is not None: - auth = parts.username - if parts.password is not None: - auth += f":{parts.password}" - netloc = f"{auth}@{netloc}" - if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get(scheme): - netloc += f":{parts.port}" # path: collapse the bare root to empty; drop a trailing slash otherwise. path = parts.path