LWLP-4918: Preserve non-UTF-8 Python sources during version rewrites

docs/migration.md

@@ -66,6 +66,11 @@ The ingestion tag is `<package>/<version>` and the baseline tag is
   and ownership. Their bytes may differ from the original Bash implementation.
 - Metadata rewrites validate paths and matches before writing. Default TOML/INI
   edits target project/metadata sections only.
+- Default Python-source version scans use ``surrogateescape`` and retain
+  universal-newline text handling. Non-UTF-8 PEP 263 sources can be scanned and
+  rewritten without rejecting the build, while changed Python files keep the
+  original newline-normalization behavior. Explicit rewrite rules continue to
+  require strict UTF-8.
 - Scratch is a reversioning-check build mode with no orchestrator equivalent: it
   skips all Git lineage queries and stamps a fixed `upstream+rhlw.0`, so a
   build-config can be exercised without ingestion/baseline tags. Whether the

src/lightwell_python/versioning.py

@@ -130,7 +130,9 @@ def rewrite_metadata(
     Raises:
         TaskError: The source is missing, a target escapes it, or an explicit rule
             does not match or fails to write the computed version.
-        UnicodeError: A metadata file cannot be decoded as UTF-8.
+        UnicodeError: An explicit rewrite target or a non-Python metadata
+            file cannot be decoded as UTF-8. Default Python-source scans use
+            ``surrogateescape`` so non-UTF-8 PEP 263 sources remain readable.
     """
     if not source.is_dir():
         raise TaskError("source/ directory is missing")
@@ -141,23 +143,36 @@ def rewrite_metadata(
     else:
         updates = _plan_default_rewrites(source, package, computed)
 
-    for path, text in updates.items():
-        path.write_text(text, encoding="utf-8")
+    for path, update in updates.items():
+        if update.python_source:
+            with path.open("w", encoding="utf-8", errors="surrogateescape", newline=None) as handle:
+                handle.write(update.text)
+        else:
+            path.write_text(update.text, encoding="utf-8")
     if not updates:
         logging.getLogger(__name__).warning(
             "No version declarations changed; wheel metadata will be verified"
         )
     return list(updates)
 
 
+@dataclass(frozen=True, slots=True)
+class _PlannedRewrite:
+    """A validated text update plus the policy needed to write it back."""
+
+    text: str
+    python_source: bool = False
+
+
 def _plan_configured_rewrites(
     source: Path, package: Package, computed: str, rules: list[dict[str, str]]
-) -> dict[Path, str]:
+) -> dict[Path, _PlannedRewrite]:
     """Validate configured replacements and collect changes without writing files."""
-    updates: dict[Path, str] = {}
+    updates: dict[Path, _PlannedRewrite] = {}
     for rule in rules:
         path = contained_file(source, rule["path"])
-        original = updates.get(path, path.read_text(encoding="utf-8"))
+        existing = updates.get(path)
+        original = existing.text if existing else path.read_text(encoding="utf-8")
         text, count = re.subn(
             rule["pattern"],
             expand(rule["replacement"], package, computed),
@@ -166,13 +181,15 @@ def _plan_configured_rewrites(
         )
         if not count or computed not in text:
             raise TaskError(f"Configured rewrite did not write computed version: {rule['path']}")
-        updates[path] = text
+        updates[path] = _PlannedRewrite(text)
     return updates
 
 
-def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dict[Path, str]:
+def _plan_default_rewrites(
+    source: Path, package: Package, computed: str
+) -> dict[Path, _PlannedRewrite]:
     """Collect changes to authoritative metadata and Python __version__ assignments."""
-    updates: dict[Path, str] = {}
+    updates: dict[Path, _PlannedRewrite] = {}
     old = re.escape(package.version)
     for candidate in sorted(source.rglob("*")):
         if candidate.is_dir():
@@ -185,7 +202,17 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic
         ):
             continue
         path = contained_file(source, str(candidate.relative_to(source)))
-        original = path.read_text(encoding="utf-8")
+        # Python sources may legitimately use non-UTF-8 encodings (PEP 263).
+        # Use surrogateescape so the heuristic can scan for an ASCII
+        # ``__version__`` declaration without rejecting the whole build, while
+        # retaining the universal-newline handling of the original text-mode read.
+        # Authoritative metadata files remain strict UTF-8.
+        python_source = candidate.suffix == ".py"
+        if python_source:
+            with path.open("r", encoding="utf-8", errors="surrogateescape", newline=None) as handle:
+                original = handle.read()
+        else:
+            original = path.read_text(encoding="utf-8")
         if name in {"PKG-INFO", "METADATA"}:
             header, separator, body = original.partition("\n\n")
             header = re.sub(
@@ -195,7 +222,7 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic
                 flags=re.MULTILINE,
             )
             text = header + separator + body
-        elif candidate.suffix == ".py":
+        elif python_source:
             text = re.sub(
                 r"""(^[ \t]*__version__[ \t]*=[ \t]*["'])""" + old + r"""(["'])""",
                 lambda m: m[1] + computed + m[2],
@@ -205,7 +232,7 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic
         else:
             text = _rewrite_sections(original, old, computed, name == "pyproject.toml")
         if text != original:
-            updates[path] = text
+            updates[path] = _PlannedRewrite(text, python_source=python_source)
     return updates
 
 

tests/unit/test_source.py

@@ -73,6 +73,76 @@ def test_rewrite_authoritative_sections(tmp_path):
     assert (tmp_path / "PKG-INFO").read_text().endswith("\n\nVersion: 1.0\n")
 
 
+def test_default_rewrite_handles_non_utf8_python_source(tmp_path):
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "pyproject.toml").write_text('[project]\nversion = "1.0"\n')
+    fixture = source / "latin1.py"
+    fixture.write_bytes(b'# coding: latin_1\n__version__ = "1.0"\nTOTO = ("Caf\xe9", "Caf\xe9")\n')
+
+    changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", [])
+
+    assert set(changes) == {source / "pyproject.toml", fixture}
+    assert b'__version__ = "1.0+rhlw.1"' in fixture.read_bytes()
+    assert b"Caf\xe9" in fixture.read_bytes()
+    assert '[project]\nversion = "1.0+rhlw.1"' in (source / "pyproject.toml").read_text()
+
+
+def test_default_rewrite_uses_scan_suffix_for_python_encoding(tmp_path):
+    source = tmp_path / "source"
+    source.mkdir()
+    target = source / "data.txt"
+    target.write_bytes(b'__version__ = "1.0"\nTOTO = "Caf\xe9"\n')
+    (source / "version.py").symlink_to(target)
+
+    changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", [])
+
+    assert changes == [target]
+    assert b'__version__ = "1.0+rhlw.1"' in target.read_bytes()
+    assert b"Caf\xe9" in target.read_bytes()
+
+
+@pytest.mark.parametrize("newline", [b"\r", b"\r\n"])
+def test_default_rewrite_normalizes_python_newlines(tmp_path, newline):
+    source = tmp_path / "source"
+    source.mkdir()
+    python_source = source / "version.py"
+    python_source.write_bytes(b'__version__ = "1.0"' + newline)
+
+    changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", [])
+
+    assert changes == [python_source]
+    assert python_source.read_bytes() == b'__version__ = "1.0+rhlw.1"\n'
+
+
+def test_explicit_rewrite_requires_utf8_python_source(tmp_path):
+    source = tmp_path / "source"
+    source.mkdir()
+    target = source / "version.py"
+    original = b'__version__ = "1.0"\nTOTO = "Caf\xe9"\n'
+    target.write_bytes(original)
+    rules = [{"path": "version.py", "pattern": "1.0", "replacement": "{COMPUTED_VERSION}"}]
+
+    with pytest.raises(UnicodeDecodeError):
+        rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", rules)
+
+    assert target.read_bytes() == original
+
+
+def test_default_rewrite_leaves_non_matching_non_utf8_python_source_unchanged(tmp_path):
+    source = tmp_path / "source"
+    source.mkdir()
+    (source / "pyproject.toml").write_text('[project]\nversion = "1.0"\n')
+    fixture = source / "latin1.py"
+    original = b'# coding: latin_1\nTOTO = "Caf\xe9"\n'
+    fixture.write_bytes(original)
+
+    changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", [])
+
+    assert changes == [source / "pyproject.toml"]
+    assert fixture.read_bytes() == original
+
+
 def test_explicit_rewrite_is_validated_before_writes(tmp_path):
     target = tmp_path / "version.txt"
     target.write_text("1.0")