diff --git a/docs/migration.md b/docs/migration.md index cf0145f..be64e45 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -66,6 +66,11 @@ The ingestion tag is `/` and the baseline tag is and ownership. Their bytes may differ from the original Bash implementation. - Metadata rewrites validate paths and matches before writing. Default TOML/INI edits target project/metadata sections only. +- Default Python-source version scans use ``surrogateescape`` and retain + universal-newline text handling. Non-UTF-8 PEP 263 sources can be scanned and + rewritten without rejecting the build, while changed Python files keep the + original newline-normalization behavior. Explicit rewrite rules continue to + require strict UTF-8. - Scratch is a reversioning-check build mode with no orchestrator equivalent: it skips all Git lineage queries and stamps a fixed `upstream+rhlw.0`, so a build-config can be exercised without ingestion/baseline tags. Whether the diff --git a/src/lightwell_python/versioning.py b/src/lightwell_python/versioning.py index 2297f9b..5507dbf 100644 --- a/src/lightwell_python/versioning.py +++ b/src/lightwell_python/versioning.py @@ -130,7 +130,9 @@ def rewrite_metadata( Raises: TaskError: The source is missing, a target escapes it, or an explicit rule does not match or fails to write the computed version. - UnicodeError: A metadata file cannot be decoded as UTF-8. + UnicodeError: An explicit rewrite target or a non-Python metadata + file cannot be decoded as UTF-8. Default Python-source scans use + ``surrogateescape`` so non-UTF-8 PEP 263 sources remain readable. """ if not source.is_dir(): raise TaskError("source/ directory is missing") @@ -141,8 +143,12 @@ def rewrite_metadata( else: updates = _plan_default_rewrites(source, package, computed) - for path, text in updates.items(): - path.write_text(text, encoding="utf-8") + for path, update in updates.items(): + if update.python_source: + with path.open("w", encoding="utf-8", errors="surrogateescape", newline=None) as handle: + handle.write(update.text) + else: + path.write_text(update.text, encoding="utf-8") if not updates: logging.getLogger(__name__).warning( "No version declarations changed; wheel metadata will be verified" @@ -150,14 +156,23 @@ def rewrite_metadata( return list(updates) +@dataclass(frozen=True, slots=True) +class _PlannedRewrite: + """A validated text update plus the policy needed to write it back.""" + + text: str + python_source: bool = False + + def _plan_configured_rewrites( source: Path, package: Package, computed: str, rules: list[dict[str, str]] -) -> dict[Path, str]: +) -> dict[Path, _PlannedRewrite]: """Validate configured replacements and collect changes without writing files.""" - updates: dict[Path, str] = {} + updates: dict[Path, _PlannedRewrite] = {} for rule in rules: path = contained_file(source, rule["path"]) - original = updates.get(path, path.read_text(encoding="utf-8")) + existing = updates.get(path) + original = existing.text if existing else path.read_text(encoding="utf-8") text, count = re.subn( rule["pattern"], expand(rule["replacement"], package, computed), @@ -166,13 +181,15 @@ def _plan_configured_rewrites( ) if not count or computed not in text: raise TaskError(f"Configured rewrite did not write computed version: {rule['path']}") - updates[path] = text + updates[path] = _PlannedRewrite(text) return updates -def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dict[Path, str]: +def _plan_default_rewrites( + source: Path, package: Package, computed: str +) -> dict[Path, _PlannedRewrite]: """Collect changes to authoritative metadata and Python __version__ assignments.""" - updates: dict[Path, str] = {} + updates: dict[Path, _PlannedRewrite] = {} old = re.escape(package.version) for candidate in sorted(source.rglob("*")): if candidate.is_dir(): @@ -185,7 +202,17 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic ): continue path = contained_file(source, str(candidate.relative_to(source))) - original = path.read_text(encoding="utf-8") + # Python sources may legitimately use non-UTF-8 encodings (PEP 263). + # Use surrogateescape so the heuristic can scan for an ASCII + # ``__version__`` declaration without rejecting the whole build, while + # retaining the universal-newline handling of the original text-mode read. + # Authoritative metadata files remain strict UTF-8. + python_source = candidate.suffix == ".py" + if python_source: + with path.open("r", encoding="utf-8", errors="surrogateescape", newline=None) as handle: + original = handle.read() + else: + original = path.read_text(encoding="utf-8") if name in {"PKG-INFO", "METADATA"}: header, separator, body = original.partition("\n\n") header = re.sub( @@ -195,7 +222,7 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic flags=re.MULTILINE, ) text = header + separator + body - elif candidate.suffix == ".py": + elif python_source: text = re.sub( r"""(^[ \t]*__version__[ \t]*=[ \t]*["'])""" + old + r"""(["'])""", lambda m: m[1] + computed + m[2], @@ -205,7 +232,7 @@ def _plan_default_rewrites(source: Path, package: Package, computed: str) -> dic else: text = _rewrite_sections(original, old, computed, name == "pyproject.toml") if text != original: - updates[path] = text + updates[path] = _PlannedRewrite(text, python_source=python_source) return updates diff --git a/tests/unit/test_source.py b/tests/unit/test_source.py index 9fcbef0..114bbe7 100644 --- a/tests/unit/test_source.py +++ b/tests/unit/test_source.py @@ -73,6 +73,76 @@ def test_rewrite_authoritative_sections(tmp_path): assert (tmp_path / "PKG-INFO").read_text().endswith("\n\nVersion: 1.0\n") +def test_default_rewrite_handles_non_utf8_python_source(tmp_path): + source = tmp_path / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\nversion = "1.0"\n') + fixture = source / "latin1.py" + fixture.write_bytes(b'# coding: latin_1\n__version__ = "1.0"\nTOTO = ("Caf\xe9", "Caf\xe9")\n') + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert set(changes) == {source / "pyproject.toml", fixture} + assert b'__version__ = "1.0+rhlw.1"' in fixture.read_bytes() + assert b"Caf\xe9" in fixture.read_bytes() + assert '[project]\nversion = "1.0+rhlw.1"' in (source / "pyproject.toml").read_text() + + +def test_default_rewrite_uses_scan_suffix_for_python_encoding(tmp_path): + source = tmp_path / "source" + source.mkdir() + target = source / "data.txt" + target.write_bytes(b'__version__ = "1.0"\nTOTO = "Caf\xe9"\n') + (source / "version.py").symlink_to(target) + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert changes == [target] + assert b'__version__ = "1.0+rhlw.1"' in target.read_bytes() + assert b"Caf\xe9" in target.read_bytes() + + +@pytest.mark.parametrize("newline", [b"\r", b"\r\n"]) +def test_default_rewrite_normalizes_python_newlines(tmp_path, newline): + source = tmp_path / "source" + source.mkdir() + python_source = source / "version.py" + python_source.write_bytes(b'__version__ = "1.0"' + newline) + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert changes == [python_source] + assert python_source.read_bytes() == b'__version__ = "1.0+rhlw.1"\n' + + +def test_explicit_rewrite_requires_utf8_python_source(tmp_path): + source = tmp_path / "source" + source.mkdir() + target = source / "version.py" + original = b'__version__ = "1.0"\nTOTO = "Caf\xe9"\n' + target.write_bytes(original) + rules = [{"path": "version.py", "pattern": "1.0", "replacement": "{COMPUTED_VERSION}"}] + + with pytest.raises(UnicodeDecodeError): + rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", rules) + + assert target.read_bytes() == original + + +def test_default_rewrite_leaves_non_matching_non_utf8_python_source_unchanged(tmp_path): + source = tmp_path / "source" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\nversion = "1.0"\n') + fixture = source / "latin1.py" + original = b'# coding: latin_1\nTOTO = "Caf\xe9"\n' + fixture.write_bytes(original) + + changes = rewrite_metadata(source, Package("demo", "1.0"), "1.0+rhlw.1", []) + + assert changes == [source / "pyproject.toml"] + assert fixture.read_bytes() == original + + def test_explicit_rewrite_is_validated_before_writes(tmp_path): target = tmp_path / "version.txt" target.write_text("1.0")