From ae1f3a5a8126a17ed5f9038e493ca83f62288268 Mon Sep 17 00:00:00 2001 From: Radim Hopp Date: Tue, 10 Nov 2026 16:21:26 +0100 Subject: [PATCH 1/2] fix: preserve non-UTF-8 Python sources during version rewrites Read and write Python candidates as bytes with surrogateescape so PEP 263 source files can be scanned and rewritten without corrupting their original bytes. Authoritative metadata files continue to use strict UTF-8. Assisted-by: Codex (GPT-5.6 Sol) --- docs/migration.md | 4 ++++ src/lightwell_python/versioning.py | 14 ++++++++++++-- tests/unit/test_source.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/migration.md b/docs/migration.md index cf0145f..2d0e6b1 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -66,6 +66,10 @@ The ingestion tag is `/` and the baseline tag is and ownership. Their bytes may differ from the original Bash implementation. - Metadata rewrites validate paths and matches before writing. Default TOML/INI edits target project/metadata sections only. +- Default Python-source version scans preserve non-UTF-8 source bytes with + ``surrogateescape``. This keeps PEP 263 fixtures intact while still allowing an + ASCII ``__version__`` declaration to be rewritten; explicit rewrite rules + continue to require strict UTF-8. - Scratch is a reversioning-check build mode with no orchestrator equivalent: it skips all Git lineage queries and stamps a fixed `upstream+rhlw.0`, so a build-config can be exercised without ingestion/baseline tags. Whether the diff --git a/src/lightwell_python/versioning.py b/src/lightwell_python/versioning.py index 2297f9b..b4fa31e 100644 --- a/src/lightwell_python/versioning.py +++ b/src/lightwell_python/versioning.py @@ -142,7 +142,10 @@ def rewrite_metadata( updates = _plan_default_rewrites(source, package, computed) for path, text in updates.items(): - path.write_text(text, encoding="utf-8") + if path.suffix == ".py": + path.write_bytes(text.encode("utf-8", "surrogateescape")) + else: + path.write_text(text, encoding="utf-8") if not updates: logging.getLogger(__name__).warning( "No version declarations changed; wheel metadata will be verified" @@ -185,7 +188,14 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic ): continue path = contained_file(source, str(candidate.relative_to(source))) - original = path.read_text(encoding="utf-8") + # Python sources may legitimately use non-UTF-8 encodings (PEP 263). + # Preserve their bytes with surrogateescape so the heuristic can scan for + # an ASCII ``__version__`` declaration without rejecting the whole build. + # Authoritative metadata files remain strict UTF-8. + if candidate.suffix == ".py": + original = path.read_bytes().decode("utf-8", "surrogateescape") + else: + original = path.read_text(encoding="utf-8") if name in {"PKG-INFO", "METADATA"}: header, separator, body = original.partition("\n\n") header = re.sub( diff --git a/tests/unit/test_source.py b/tests/unit/test_source.py index 9fcbef0..6a8d677 100644 --- a/tests/unit/test_source.py +++ b/tests/unit/test_source.py @@ -73,6 +73,21 @@ def test_rewrite_authoritative_sections(tmp_path): assert (tmp_path / "PKG-INFO").read_text().endswith("\n\nVersion: 1.0\n") +def test_default_rewrite_handles_non_utf8_python_source(tmp_path): + source = tmp_path / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\nversion = "1.0"\n') + fixture = source / "latin1.py" + fixture.write_bytes(b'# coding: latin_1\n__version__ = "1.0"\nTOTO = ("Caf\xe9", "Caf\xe9")\n') + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert set(changes) == {source / "pyproject.toml", fixture} + assert b'__version__ = "1.0+rhlw.1"' in fixture.read_bytes() + assert b"Caf\xe9" in fixture.read_bytes() + assert '[project]\nversion = "1.0+rhlw.1"' in (source / "pyproject.toml").read_text() + + def test_explicit_rewrite_is_validated_before_writes(tmp_path): target = tmp_path / "version.txt" target.write_text("1.0") From 59b8e9469a6e975b530d7c2eb05302692d86a8f9 Mon Sep 17 00:00:00 2001 From: Radim Hopp Date: Tue, 10 Nov 2026 16:29:18 +0100 Subject: [PATCH 2/2] fix: address non-UTF-8 rewrite review findings Carry the Python-source policy with each planned rewrite so symlinked paths use the same read and write handling. Restore universal-newline normalization for Python sources and add regression coverage for symlinked non-UTF-8 files, CR/CRLF inputs, strict explicit rules, and unchanged fixtures. Assisted-by: Codex (GPT-5.6 Sol) --- docs/migration.md | 9 ++--- src/lightwell_python/versioning.py | 51 ++++++++++++++++++--------- tests/unit/test_source.py | 55 ++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/docs/migration.md b/docs/migration.md index 2d0e6b1..be64e45 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -66,10 +66,11 @@ The ingestion tag is `/` and the baseline tag is and ownership. Their bytes may differ from the original Bash implementation. - Metadata rewrites validate paths and matches before writing. Default TOML/INI edits target project/metadata sections only. -- Default Python-source version scans preserve non-UTF-8 source bytes with - ``surrogateescape``. This keeps PEP 263 fixtures intact while still allowing an - ASCII ``__version__`` declaration to be rewritten; explicit rewrite rules - continue to require strict UTF-8. +- Default Python-source version scans use ``surrogateescape`` and retain + universal-newline text handling. Non-UTF-8 PEP 263 sources can be scanned and + rewritten without rejecting the build, while changed Python files keep the + original newline-normalization behavior. Explicit rewrite rules continue to + require strict UTF-8. - Scratch is a reversioning-check build mode with no orchestrator equivalent: it skips all Git lineage queries and stamps a fixed `upstream+rhlw.0`, so a build-config can be exercised without ingestion/baseline tags. Whether the diff --git a/src/lightwell_python/versioning.py b/src/lightwell_python/versioning.py index b4fa31e..5507dbf 100644 --- a/src/lightwell_python/versioning.py +++ b/src/lightwell_python/versioning.py @@ -130,7 +130,9 @@ def rewrite_metadata( Raises: TaskError: The source is missing, a target escapes it, or an explicit rule does not match or fails to write the computed version. - UnicodeError: A metadata file cannot be decoded as UTF-8. + UnicodeError: An explicit rewrite target or a non-Python metadata + file cannot be decoded as UTF-8. Default Python-source scans use + ``surrogateescape`` so non-UTF-8 PEP 263 sources remain readable. """ if not source.is_dir(): raise TaskError("source/ directory is missing") @@ -141,11 +143,12 @@ def rewrite_metadata( else: updates = _plan_default_rewrites(source, package, computed) - for path, text in updates.items(): - if path.suffix == ".py": - path.write_bytes(text.encode("utf-8", "surrogateescape")) + for path, update in updates.items(): + if update.python_source: + with path.open("w", encoding="utf-8", errors="surrogateescape", newline=None) as handle: + handle.write(update.text) else: - path.write_text(text, encoding="utf-8") + path.write_text(update.text, encoding="utf-8") if not updates: logging.getLogger(__name__).warning( "No version declarations changed; wheel metadata will be verified" @@ -153,14 +156,23 @@ def rewrite_metadata( return list(updates) +@dataclass(frozen=True, slots=True) +class _PlannedRewrite: + """A validated text update plus the policy needed to write it back.""" + + text: str + python_source: bool = False + + def _plan_configured_rewrites( source: Path, package: Package, computed: str, rules: list[dict[str, str]] -) -> dict[Path, str]: +) -> dict[Path, _PlannedRewrite]: """Validate configured replacements and collect changes without writing files.""" - updates: dict[Path, str] = {} + updates: dict[Path, _PlannedRewrite] = {} for rule in rules: path = contained_file(source, rule["path"]) - original = updates.get(path, path.read_text(encoding="utf-8")) + existing = updates.get(path) + original = existing.text if existing else path.read_text(encoding="utf-8") text, count = re.subn( rule["pattern"], expand(rule["replacement"], package, computed), @@ -169,13 +181,15 @@ def _plan_configured_rewrites( ) if not count or computed not in text: raise TaskError(f"Configured rewrite did not write computed version: {rule['path']}") - updates[path] = text + updates[path] = _PlannedRewrite(text) return updates -def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dict[Path, str]: +def _plan_default_rewrites( + source: Path, package: Package, computed: str +) -> dict[Path, _PlannedRewrite]: """Collect changes to authoritative metadata and Python __version__ assignments.""" - updates: dict[Path, str] = {} + updates: dict[Path, _PlannedRewrite] = {} old = re.escape(package.version) for candidate in sorted(source.rglob("*")): if candidate.is_dir(): @@ -189,11 +203,14 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic continue path = contained_file(source, str(candidate.relative_to(source))) # Python sources may legitimately use non-UTF-8 encodings (PEP 263). - # Preserve their bytes with surrogateescape so the heuristic can scan for - # an ASCII ``__version__`` declaration without rejecting the whole build. + # Use surrogateescape so the heuristic can scan for an ASCII + # ``__version__`` declaration without rejecting the whole build, while + # retaining the universal-newline handling of the original text-mode read. # Authoritative metadata files remain strict UTF-8. - if candidate.suffix == ".py": - original = path.read_bytes().decode("utf-8", "surrogateescape") + python_source = candidate.suffix == ".py" + if python_source: + with path.open("r", encoding="utf-8", errors="surrogateescape", newline=None) as handle: + original = handle.read() else: original = path.read_text(encoding="utf-8") if name in {"PKG-INFO", "METADATA"}: @@ -205,7 +222,7 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic flags=re.MULTILINE, ) text = header + separator + body - elif candidate.suffix == ".py": + elif python_source: text = re.sub( r"""(^[ \t]*__version__[ \t]*=[ \t]*["'])""" + old + r"""(["'])""", lambda m: m[1] + computed + m[2], @@ -215,7 +232,7 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic else: text = _rewrite_sections(original, old, computed, name == "pyproject.toml") if text != original: - updates[path] = text + updates[path] = _PlannedRewrite(text, python_source=python_source) return updates diff --git a/tests/unit/test_source.py b/tests/unit/test_source.py index 6a8d677..114bbe7 100644 --- a/tests/unit/test_source.py +++ b/tests/unit/test_source.py @@ -88,6 +88,61 @@ def test_default_rewrite_handles_non_utf8_python_source(tmp_path): assert '[project]\nversion = "1.0+rhlw.1"' in (source / "pyproject.toml").read_text() +def test_default_rewrite_uses_scan_suffix_for_python_encoding(tmp_path): + source = tmp_path / "source" + source.mkdir() + target = source / "data.txt" + target.write_bytes(b'__version__ = "1.0"\nTOTO = "Caf\xe9"\n') + (source / "version.py").symlink_to(target) + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert changes == [target] + assert b'__version__ = "1.0+rhlw.1"' in target.read_bytes() + assert b"Caf\xe9" in target.read_bytes() + + +@pytest.mark.parametrize("newline", [b"\r", b"\r\n"]) +def test_default_rewrite_normalizes_python_newlines(tmp_path, newline): + source = tmp_path / "source" + source.mkdir() + python_source = source / "version.py" + python_source.write_bytes(b'__version__ = "1.0"' + newline) + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert changes == [python_source] + assert python_source.read_bytes() == b'__version__ = "1.0+rhlw.1"\n' + + +def test_explicit_rewrite_requires_utf8_python_source(tmp_path): + source = tmp_path / "source" + source.mkdir() + target = source / "version.py" + original = b'__version__ = "1.0"\nTOTO = "Caf\xe9"\n' + target.write_bytes(original) + rules = [{"path": "version.py", "pattern": "1.0", "replacement": "{COMPUTED_VERSION}"}] + + with pytest.raises(UnicodeDecodeError): + rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", rules) + + assert target.read_bytes() == original + + +def test_default_rewrite_leaves_non_matching_non_utf8_python_source_unchanged(tmp_path): + source = tmp_path / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\nversion = "1.0"\n') + fixture = source / "latin1.py" + original = b'# coding: latin_1\nTOTO = "Caf\xe9"\n' + fixture.write_bytes(original) + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert changes == [source / "pyproject.toml"] + assert fixture.read_bytes() == original + + def test_explicit_rewrite_is_validated_before_writes(tmp_path): target = tmp_path / "version.txt" target.write_text("1.0")