From 879590ef3b83d8becfb551a890c49309b10661fb Mon Sep 17 00:00:00 2001 From: Jaden Earl Date: Mon, 3 Aug 2026 21:16:22 -0600 Subject: [PATCH 1/2] Guard lazy urlsplit port/hostname errors in canonicalize_url Co-Authored-By: Claude Fable 5 --- .../test_source_archive/test_canonicalize.py | 10 +++++ .../source_archive/canonicalize.py | 40 +++++++++++-------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py index e9476409..e45405c0 100644 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py +++ b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_canonicalize.py @@ -70,3 +70,13 @@ def test_distinct_pages_keep_distinct_hashes(): def test_empty_and_none_safe(): assert canonicalize_url("") == "" + + +def test_lazy_port_valueerror_returns_raw() -> None: + # .port raises ValueError lazily when the port section is non-numeric — + # junk the URL regex can extract from CSS-ish text ("Port could not be + # cast to integer value as 'root{--novem-render-frac'"). canonicalize + # must not propagate it. + junk = "http://a.test:root{--novem-render-frac/x" + assert canonicalize_url(junk) == junk + assert canonicalize_url(canonicalize_url(junk)) == canonicalize_url(junk) diff --git a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py index b791a47e..4139c41e 100644 --- a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py +++ b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py @@ -73,27 +73,33 @@ def canonicalize_url(url: str) -> str: if not url: return url raw = url.strip() + # urlsplit() itself rarely raises; .hostname/.port are LAZY properties that + # raise ValueError on junk like "http://root{--x:80/" or bad IPv6 — so the + # guard must cover the whole netloc normalization, not just the split. try: parts = urlsplit(raw) + if parts.scheme not in ("http", "https") or not parts.netloc: + return raw + + scheme = parts.scheme.lower() + + # netloc: lowercase host (bracket IPv6), keep userinfo, strip default + # port. + host = (parts.hostname or "").lower() + if ":" in host: # IPv6 literal + host = f"[{host}]" + netloc = host + if parts.username is not None: + auth = parts.username + if parts.password is not None: + auth += f":{parts.password}" + netloc = f"{auth}@{netloc}" + if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get( + scheme + ): + netloc += f":{parts.port}" except ValueError: return raw - if parts.scheme not in ("http", "https") or not parts.netloc: - return raw - - scheme = parts.scheme.lower() - - # netloc: lowercase host (bracket IPv6), keep userinfo, strip default port. - host = (parts.hostname or "").lower() - if ":" in host: # IPv6 literal - host = f"[{host}]" - netloc = host - if parts.username is not None: - auth = parts.username - if parts.password is not None: - auth += f":{parts.password}" - netloc = f"{auth}@{netloc}" - if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get(scheme): - netloc += f":{parts.port}" # path: collapse the bare root to empty; drop a trailing slash otherwise. path = parts.path From 0db6be15eb9e351d2e1229b507c41f35ec412ea4 Mon Sep 17 00:00:00 2001 From: Jaden Earl Date: Tue, 4 Aug 2026 10:19:22 -0600 Subject: [PATCH 2/2] Apply pre-commit formatting --- .../agents_and_tools/source_archive/canonicalize.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py index 4139c41e..7d722c69 100644 --- a/forecasting_tools/agents_and_tools/source_archive/canonicalize.py +++ b/forecasting_tools/agents_and_tools/source_archive/canonicalize.py @@ -94,9 +94,7 @@ def canonicalize_url(url: str) -> str: if parts.password is not None: auth += f":{parts.password}" netloc = f"{auth}@{netloc}" - if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get( - scheme - ): + if parts.port is not None and str(parts.port) != _DEFAULT_PORTS.get(scheme): netloc += f":{parts.port}" except ValueError: return raw