From b1bc81500680a3c5b2b8e9ca22786d82c0ee5b61 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 08:51:04 +0100 Subject: [PATCH 01/21] docs(tree): clarify supported cache editing behavior (#1771) `Tree.cache` and `TreeModifier` were announced as removed in an old changelog, but remain exported and supported. Their documentation also suggested sorting alone would make edits persistent, leaving their status and practical limits unclear. Document that cache edits affect one instance and do not update its object ID, stored tree, parents, index, or commits. Correct the historical removal claim and recommend `IndexFile.write_tree()` for staged path changes. Serialization already delegates construction to Git or Gix; no deprecation or replacement implementation is needed. Git's `Documentation/git-mktree.adoc` confirms that `mktree` normalizes entry order without pre-sorted input. Add a regression covering local subtree edits, serialization into a new object, and unchanged parent references. The focused tree tests passed with both CLI and Gix backends (3 tests each), using Git 2.54.0 and GixPython 0.3.0. The HTML build rendered the updated documentation; its strict check reported one existing field-list warning in `git.objects.submodule.util.is_relative_url`. --- doc/source/changes.rst | 9 ++++++++- git/objects/tree.py | 23 ++++++++++++++--------- test/test_tree.py | 23 ++++++++++++++++++++++- tests/fixtures/tree-cache.sh | 7 +++++++ 4 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 tests/fixtures/tree-cache.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 78d55ba01..b93095f9b 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -51,6 +51,10 @@ Git executable and safety API changes ----------- +* ``Tree.cache`` and ``TreeModifier`` remain supported. They edit one tree's + in-memory entries; object IDs and parent trees do not change automatically. + Tree serialization now delegates construction to Git or the supported Gix + backend. Prefer ``IndexFile.write_tree()`` for changes to staged paths. * ``Commit.stats`` preserves Unicode line separators in filenames when ``core.quotePath=false``. Git's LF-delimited output remains intact for root, subsequent and empty commits, including the Gix backend's existing filename @@ -1135,7 +1139,10 @@ Please note that due to breaking changes, we have to increase the major version. * CRITICAL: fixed incorrect `Commit` object serialization when authored or commit date had timezones which were not divisiblej by 3600 seconds. This would happen if the timezone was something like `+0530` for instance. * A list of all additional fixes can be found `on GitHub `__ -* CRITICAL: `Tree.cache` was removed without replacement. It is technically impossible to change individual trees and expect their serialization results to be consistent with what *git* expects. Instead, use the `IndexFile` facilities to adjust the content of the staging area, and write it out to the respective tree objects using `IndexFile.write_tree()` instead. +* Historical correction: ``Tree.cache`` was announced as removed, but the interface + remained available. Its current behavior and limitations are documented in + :class:`~git.objects.tree.TreeModifier`. Prefer ``IndexFile.write_tree()`` when + building trees from the staging area. 1.0.1 - Fixes ============= diff --git a/git/objects/tree.py b/git/objects/tree.py index 6a469284c..9c840b3a1 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -55,12 +55,17 @@ def cmp(a: str, b: str) -> int: class TreeModifier: - """A utility class providing methods to alter the underlying cache in a list-like - fashion. + """Edit the entries cached by one :class:`Tree` instance. - Once all adjustments are complete, the :attr:`_cache`, which really is a reference - to the cache of a tree, will be sorted. This ensures it will be in a serializable - state. + This interface is supported; it is not deprecated. Changes affect only that + instance's in-memory entries. They do not update its object ID, the stored tree, + parent trees, the index, or any commit. Call :meth:`set_done` to restore entry + order for iteration after editing. + + Tree serialization delegates construction and ordering to Git or the supported + Gix backend. Prefer :class:`~git.index.base.IndexFile` and + :meth:`~git.index.base.IndexFile.write_tree` when building a new tree from staged + paths, especially when changes must propagate through nested trees. """ __slots__ = ("_cache",) @@ -303,10 +308,10 @@ def blobs(self) -> List[Blob]: def cache(self) -> TreeModifier: """ :return: - An object allowing modification of the internal cache. This can be used to - change the tree's contents. When done, make sure you call - :meth:`~TreeModifier.set_done` on the tree modifier, or serialization - behaviour will be incorrect. + A supported editor for this instance's cached entries. Call + :meth:`~TreeModifier.set_done` after editing to restore iteration order. + Editing does not change this tree's object ID or persist changes to its + stored object, parents, or commits. :note: See :class:`TreeModifier` for more information on how to alter the cache. diff --git a/test/test_tree.py b/test/test_tree.py index e60b355a3..5c46027ec 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -15,11 +15,32 @@ from git.objects import Blob, Tree from git.objects.tree import TreeModifier from test.lib import TestBase, with_rw_repo -from test.lib.fixtures import read_only_fixture +from test.lib.fixtures import fixture_repo, read_only_fixture from .lib.helper import PathLikeMock +def test_tree_cache_edits_do_not_update_object_ids_or_parents(): + with fixture_repo("tree-cache.sh", write=True) as repo: + root = repo.head.commit.tree + subtree = root["sub"] + original_oid = subtree.hexsha + del subtree.cache["file"] + subtree.cache.set_done() + assert not list(subtree) + assert subtree.hexsha == original_oid + assert root["sub"].hexsha == original_oid + assert [item.name for item in repo.head.commit.tree["sub"]] == ["file"] + + data = BytesIO() + subtree._serialize(data) + data.seek(0) + stored = repo.odb.store(IStream("tree", len(data.getvalue()), data)) + assert stored.binsha != subtree.binsha + assert not list(Tree(repo, stored.binsha, path="")) + assert repo.head.commit.tree["sub"].hexsha == original_oid + + @ddt.ddt class TestTree(TestBase): @with_rw_repo("HEAD") diff --git a/tests/fixtures/tree-cache.sh b/tests/fixtures/tree-cache.sh new file mode 100644 index 000000000..f5fb4c0d1 --- /dev/null +++ b/tests/fixtures/tree-cache.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main repo +mkdir repo/sub +printf 'original\n' >repo/sub/file +git -C repo add sub/file +git -C repo commit -m initial From ee2cd6faae6833cd19c36a119f30476de57ecb9c Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 08:52:36 +0100 Subject: [PATCH 02/21] docs: render supported subclass interface docstrings (#1848) Sphinx omits underscore-prefixed methods by default, hiding documented extension points such as `RemoteProgress._parse_progress_line()` and `LockFile._obtain_lock()`. Traversal documentation used a separate allowlist, and `Tree.traverse()` linked to its implementation using an ordinary URL. Mark supported subclass interfaces with Sphinx's built-in `:meta public:` docstring metadata. Remove the traversal-specific `:private-members:` list, use a method reference for traversal documentation, and explain this convention in `CONTRIBUTING.md`. Private implementation helpers stay hidden. Repair an existing malformed `is_relative_url()` return field so strict HTML validation can complete. Before this change, the HTML reference lacked the progress and lock extension-point targets. The strict Sphinx HTML build now passes. Assertions against its output verify all five selected interface targets, the traversal link, and omission of `LockFile._lock_file_path()`. Runtime logic is unchanged. --- CONTRIBUTING.md | 6 ++++++ doc/source/reference.rst | 1 - git/objects/submodule/util.py | 7 +++++-- git/objects/tree.py | 3 +-- git/objects/util.py | 6 ++++++ git/util.py | 4 ++++ 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a66f60c5..fa3d6bc39 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,12 @@ Attributing AI assistance in commit metadata, for example with a `Co-authored-by trailer, is welcome but not required. Code is reviewed the same way regardless of its origin. +## Documenting subclass interfaces + +Use Sphinx's `:meta public:` field in docstrings of underscore-prefixed methods +that form a supported interface for subclasses. This includes their documentation +in the API reference without exposing unrelated private implementation details. + ## Temporary test directories Tests require Python 3.11+, Git 2.52+, Bash, and the `test` extra, which installs diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 49d732ca2..13dd38d02 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -91,7 +91,6 @@ Objects.Util .. automodule:: git.objects.util :members: :undoc-members: - :private-members: _get_intermediate_items, _list_traverse, _traverse :special-members: Index.Base diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index a9cccc402..067f2ea58 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -99,8 +99,11 @@ def _find_last_dir_sep(url: str) -> int: def is_relative_url(url: str) -> bool: - """:return: ``True`` if `url` starts with ``./`` or ``../``, which git resolves - against the superproject's remote rather than treating as a path""" + """Check whether Git resolves a URL against the superproject's remote. + + :return: + ``True`` if `url` starts with ``./`` or ``../``. + """ return _starts_with_dot_slash(url) or _starts_with_dot_dot_slash(url) diff --git a/git/objects/tree.py b/git/objects/tree.py index 9c840b3a1..f04e0d75c 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -328,8 +328,7 @@ def traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]]: - """For documentation, see - `Traversable._traverse() `. + """For documentation, see :meth:`~git.objects.util.Traversable._traverse`. Trees are set to ``visit_once = False`` to gain more performance in the traversal. diff --git a/git/objects/util.py b/git/objects/util.py index 05f2955ab..e24aeb6ae 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -334,6 +334,8 @@ class Traversable(Protocol): @abstractmethod def _get_intermediate_items(cls, item: Any) -> Sequence["Traversable"]: """ + :meta public: + :return: Tuple of items connected to the given item. Must be implemented in subclass. @@ -354,6 +356,8 @@ def _list_traverse( ) -> IterableList[Union["Commit", "Submodule", "Tree", "Blob"]]: """Traverse self and collect all items found. + :meta public: + :return: :class:`~git.util.IterableList` with the results of the traversal as produced by :meth:`traverse`:: @@ -401,6 +405,8 @@ def _traverse( ) -> Union[Iterator[Union["Traversable", "Blob"]], Iterator[TraversedTup]]: """Iterator yielding items found when traversing `self`. + :meta public: + :param predicate: A function ``f(i,d)`` that returns ``False`` if item i at depth ``d`` should not be included in the result. diff --git a/git/util.py b/git/util.py index bd2fcf916..8d6a57530 100644 --- a/git/util.py +++ b/git/util.py @@ -712,6 +712,8 @@ def _parse_progress_line(self, line: AnyStr) -> Optional[object]: """Parse progress information from the given line as retrieved by :manpage:`git-push(1)` or :manpage:`git-fetch(1)`. + :meta public: + - Lines that do not contain progress info are stored in :attr:`other_lines`. - Lines that seem to contain an error (i.e. start with ``error:`` or ``fatal:``) are stored in :attr:`error_lines`. @@ -1171,6 +1173,8 @@ def _obtain_lock_or_raise(self) -> None: def _obtain_lock(self) -> None: """The default implementation will raise if a lock cannot be obtained. + :meta public: + Subclasses may override this method to provide a different implementation. """ return self._obtain_lock_or_raise() From a28ad51da98bb8fc1004c22dadb10cdcf51c4caf Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 08:54:46 +0100 Subject: [PATCH 03/21] fix(diff): preserve legacy filename bytes in staged diffs (#2030) Aider-style staged-file reads fail with `UnicodeDecodeError` when a raw diff contains a non-UTF-8 filename. Index entries and blob streams already preserve the legacy bytes, but `Diff._handle_diff_line()` decoded the entire NUL-delimited output before separating metadata from paths. Keep raw diff fields as bytes, decode only the ASCII mode/OID/status metadata, and pass original path bytes into `Diff`. This also preserves rename and copy destinations without requiring callers to reencode them. The existing text path properties retain their display behavior. Git 2.54.0's `Documentation/diff-options.adoc` specifies that `--raw -z` returns unmodified pathnames separated by NULs. All six new regressions failed before the fix: five cover raw add/delete/ modify/rename/copy records, and one stages Windows-1252 filename and content bytes in a real index and reads its diff and blobs. All diff and index tests now pass with CLI and Gix backends: 150 passed, 2 skipped, and 9 subtests on each. Ruff and ShellCheck pass. Windows execution remains for CI validation; no conversion of the legacy files is needed. --- doc/source/changes.rst | 3 +++ git/diff.py | 19 +++++++------------ test/test_diff.py | 17 +++++++++++++++++ test/test_index.py | 14 ++++++++++++++ tests/fixtures/legacy-index.sh | 9 +++++++++ 5 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/legacy-index.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index b93095f9b..7c7d8c7fe 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -51,6 +51,9 @@ Git executable and safety API changes ----------- +* Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``, + and raw rename fields. Staged-file diffs no longer fail while decoding legacy + filenames; blob contents remain byte streams regardless of their encoding. * ``Tree.cache`` and ``TreeModifier`` remain supported. They edit one tree's in-memory entries; object IDs and parent trees do not change automatically. Tree serialization now delegates construction to Git or the supported Gix diff --git a/git/diff.py b/git/diff.py index 7a690c50c..a5c1a91b9 100644 --- a/git/diff.py +++ b/git/diff.py @@ -761,34 +761,29 @@ def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> N @staticmethod def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]) -> None: - lines = lines_bytes.decode(defenc) - # Discard everything before the first colon, and the colon itself. - _, _, lines = lines.partition(":") + _, _, lines = lines_bytes.partition(b":") # Each record is a metadata field starting with a colon, followed by one path, # or two for copies and renames, each terminated by NUL. A path may itself # start with a colon or contain a line feed, so read the fields in order # instead of splitting at colons. - fields = iter(lines.rstrip("\n").split("\x00")) + fields = iter(lines.rstrip(b"\n").split(b"\x00")) for meta in fields: if not meta: # The line data is empty, skip. continue - if meta.startswith(":"): - meta = meta[1:] + metadata = (meta[1:] if meta.startswith(b":") else meta).decode("ascii") a_blob_id: Optional[str] b_blob_id: Optional[str] - old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4) + old_mode, new_mode, a_blob_id, b_blob_id, _change_type = metadata.split(None, 4) # Change type can be R100 # R: status letter # 100: score (in case of copy and rename) change_type: Lit_change_type = cast(Lit_change_type, _change_type[0]) score_str = "".join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None - path = next(fields, "") - a_path = path.encode(defenc) - b_path = path.encode(defenc) + a_path = b_path = next(fields, b"") deleted_file = False new_file = False copied_file = False @@ -805,9 +800,9 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"] new_file = True elif change_type == "C": copied_file = True - b_path = next(fields, "").encode(defenc) + b_path = next(fields, b"") elif change_type == "R": - b_path = next(fields, "").encode(defenc) + b_path = next(fields, b"") rename_from, rename_to = a_path, b_path elif change_type == "T": # Nothing to do. diff --git a/test/test_diff.py b/test/test_diff.py index ea15ad157..9426b78fc 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -6,6 +6,7 @@ import os.path as osp import ddt +import pytest from git import NULL_TREE, Diff, DiffIndex, Diffable, Repo from git.diff import decode_path @@ -19,6 +20,22 @@ def to_raw(input): return input.replace(b"\t", b"\x00") +@pytest.mark.parametrize("change", [b"A", b"D", b"M", b"R100", b"C100"]) +def test_raw_diff_preserves_non_utf8_paths(change): + before = b"legacy-\xf5\t:\n.pas" + after = b"renamed-\xe3.pas" + output = b":100644 100644 " + b"1" * 40 + b" " + b"2" * 40 + b" " + change + b"\0" + before + b"\0" + if change.startswith((b"R", b"C")): + output += after + b"\0" + diffs = DiffIndex() + Diff._handle_diff_line(output, None, diffs) + assert len(diffs) == 1 + assert diffs[0].a_rawpath == before + assert diffs[0].b_rawpath == (after if change.startswith((b"R", b"C")) else before) + if change.startswith(b"R"): + assert (diffs[0].raw_rename_from, diffs[0].raw_rename_to) == (before, after) + + @ddt.ddt class TestDiff(TestBase): def setUp(self): diff --git a/test/test_index.py b/test/test_index.py index f5c0ce8ef..d4db16ffb 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -18,6 +18,7 @@ from git import NULL_TREE, BlobFilter, Diff, Git, IndexFile, Object, Repo, Tree from git.diff import NULL_TREE_SHA +from git.compat import defenc from git.exc import ( CheckoutError, GitCommandError, @@ -38,6 +39,19 @@ _logger = logging.getLogger(__name__) +def test_staged_legacy_filename_and_blob_bytes(): + with fixture_repo("legacy-index.sh") as repo: + (entry,) = repo.index.iter_entries() + assert entry.path.encode(defenc, "surrogateescape") == b"fonte-\xf5.pas" + assert repo.odb.stream(entry.binsha).read() == b"// vers\xe3o nova\n" + (diff,) = repo.index.diff("HEAD") + assert (diff.a_rawpath, diff.b_rawpath) == (b"fonte-\xf5.pas", b"fonte-\xf5.pas") + assert {blob.data_stream.read() for blob in (diff.a_blob, diff.b_blob)} == { + b"// vers\xe3o original\n", + b"// vers\xe3o nova\n", + } + + @ddt.ddt class TestIndex(TestBase): @with_rw_repo("HEAD") diff --git a/tests/fixtures/legacy-index.sh b/tests/fixtures/legacy-index.sh new file mode 100644 index 000000000..e33c433d5 --- /dev/null +++ b/tests/fixtures/legacy-index.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main repo +cd repo +original=$(printf '// vers\343o original\n' | git hash-object -w --stdin) +printf '100644 %s\tfonte-\365.pas\0' "$original" | git update-index -z --index-info +git commit -m initial +staged=$(printf '// vers\343o nova\n' | git hash-object -w --stdin) +printf '100644 %s\tfonte-\365.pas\0' "$staged" | git update-index -z --index-info From 70d2fd5d218c7522c6c7fb59ecc3a06d4de79ae3 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 08:59:01 +0100 Subject: [PATCH 04/21] fix(refs): preserve non-UTF-8 names at native API boundaries (#2064) Packed-reference enumeration now delegates to Git/Gix and no longer has the original UTF-8 file-reader failure. However, tag paths retain undecodable bytes as surrogate escapes, and passing those strings back into GixPython raises `UnicodeEncodeError` before validation or reference lookup. Make `safe_encode()` the byte-preserving inverse of `safe_decode()`. Use it at the supported Gix reference validation, lookup, revision, reflog and edit boundaries, and when serializing Git reference transactions. GixPython 0.3.0 accepts bytes in these APIs, so this fixes adapter conversion without adding a reference-storage implementation or requiring an upstream workaround. Add a scripted packed-ref fixture with lightweight and annotated non-UTF-8 tag names for both SHA-1 and SHA-256. Git 2.54.0's `for-each-ref` supplies the reference inventory; `git-for-each-ref.adoc` documents the `refname` and `objectname` projections used for the baseline. The regression checks byte round trips, tag/commit resolution, CLI parity and absence of CLI fallback during native resolution. Packed records are fixture inputs because APFS rejects those bytes in loose reference filenames. Validation: 42 reference tests and 24 subtests passed with CLI; 144 reference and backend tests and 24 subtests passed with Gix. Ruff, ShellCheck and mypy passed. Windows behavior will be validated by CI. --- doc/gix-backend.md | 9 +++++++++ doc/source/changes.rst | 3 +++ git/_backend.py | 32 ++++++++++++++++++-------------- git/compat.py | 4 ++-- git/refs/symbolic.py | 3 ++- test/test_refs.py | 26 ++++++++++++++++++++++++++ tests/fixtures/legacy-refs.sh | 11 +++++++++++ 7 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 tests/fixtures/legacy-refs.sh diff --git a/doc/gix-backend.md b/doc/gix-backend.md index 677cc5542..6e44ab852 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -389,6 +389,15 @@ repository-format validation gap. | `Repo.untracked_files` | Native directory walk | Extra status options and non-root command directories | | `Repo.ignored` | Native excludes with tracked-file suppression | Symlink/submodule traversal, unsupported path normalization, and unavailable path metadata | +Reference names and revisions are encoded with `surrogateescape` before calling +GixPython's byte-capable `Target.Symbolic`, `Repository.try_find_reference`, +`Repository.rev_parse_single`, `Repository.rev_parse` and `RefEdit` APIs. +This preserves non-UTF-8 packed tag names without interpreting reference storage +in Python. `test_non_utf8_tag_names_preserve_bytes_and_resolve` covers lightweight +and annotated tags in SHA-1 and SHA-256 repositories and forbids CLI fallback +during native resolution. This corrects adapter input conversion; GixPython 0.3.0 +already accepts the original bytes through those APIs. + Repository-backed native operations fall back for reftable (GIX-1) and repositories with `extensions.compatObjectFormat` (GIX-19). Process-returning calls, timeouts, custom storage/environment, global Git options, and unhandled command options diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 7c7d8c7fe..65f03c548 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -51,6 +51,9 @@ Git executable and safety API changes ----------- +* Packed tag names containing non-UTF-8 bytes remain losslessly represented as + surrogate-escaped strings. ``safe_encode()`` recovers their original bytes, + and Gix reference validation and resolution receive those bytes directly. * Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``, and raw rename fields. Staged-file diffs no longer fail while decoding legacy filenames; blob contents remain byte streams regardless of their encoding. diff --git a/git/_backend.py b/git/_backend.py index ab32bb450..9d5fb2b45 100644 --- a/git/_backend.py +++ b/git/_backend.py @@ -16,7 +16,7 @@ from threading import Lock from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, cast -from git.compat import defenc, safe_decode +from git.compat import defenc, safe_decode, safe_encode from git.exc import GitCommandError _selection = os.environ.get("GIT_PYTHON_BACKEND") @@ -218,7 +218,7 @@ def _oid(repo: Any, ref: Any, env: Optional[Dict[str, Any]] = None) -> Any: _index(repo, {"env": env or {}}) if not hasattr(repo, "rev_parse_single"): raise _Unsupported("revision feature disabled") - return repo.rev_parse_single(ref) + return repo.rev_parse_single(safe_encode(ref)) def object_data(command: Any, ref: bytes, *, stream: bool = False) -> Any: @@ -296,7 +296,7 @@ def revision_info(command: Any, ref: str) -> Any: repo = _repository(command, {}) if ref.startswith(":"): _index(repo, {"env": command.environment()}) - spec = repo.rev_parse(ref) + spec = repo.rev_parse(safe_encode(ref)) oid = spec.single() if oid is None: raise _Unsupported("multiple revisions") @@ -362,7 +362,7 @@ def check_ref_name(name: str) -> Any: if gix is None: return NotImplemented try: - gix.Target.Symbolic(name) + gix.Target.Symbolic(safe_encode(name)) except gix.Error as exc: if "/" not in name: return _fallback("Reference.validate", "standalone reference names (GIX-20)") @@ -380,10 +380,11 @@ def reference_info(command: Any, path: str) -> Any: if path != "HEAD" and not path.startswith("refs/"): raise _Unsupported("partial reference names") repo = _repository(command, {}) - reference = repo.try_find_reference(path) + encoded_path = safe_encode(path) + reference = repo.try_find_reference(encoded_path) result = None if reference is not None: - if reference.name() != os.fsencode(path): + if reference.name() != encoded_path: raise _Unsupported("partial reference lookup") target = reference.target() name = target.try_name() @@ -400,7 +401,7 @@ def reference_info(command: Any, path: str) -> Any: def _symbolic_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: if len(args) != 4 or args[:3] != ["--quiet", "--no-recurse", "--"]: raise _Unsupported("reference mutation or options") - reference = repo.try_find_reference(args[-1]) + reference = repo.try_find_reference(safe_encode(args[-1])) if reference is None: raise _Unsupported("missing reference diagnostics") target = reference.target().try_name() @@ -415,7 +416,7 @@ def _for_each_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: split = args.index("--") if "--" in args else len(args) if args[1:split] or len(args[split + 1 :]) > 1: raise _Unsupported("root refs or reference options") - prefix = os.fsencode(args[-1]) if len(args) > split + 1 else b"" + prefix = safe_encode(args[-1]) if len(args) > split + 1 else b"" if any(char in prefix for char in b"*?["): raise _Unsupported("reference glob patterns") with repo.references().all() as refs: @@ -739,7 +740,7 @@ def _date(signature: Any) -> bytes: def _reflog(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: if len(args) == 3 and args[:2] == ["exists", "--"]: - reference = repo.try_find_reference(args[-1]) + reference = repo.try_find_reference(safe_encode(args[-1])) if reference is None: raise _Unsupported("orphan reflog lookup") if not reference.log_exists(): @@ -761,7 +762,7 @@ def _reflog(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: or args[-1] != "--" ): raise _Unsupported("reflog writing or format") - reference = repo.try_find_reference(args[-2]) + reference = repo.try_find_reference(safe_encode(args[-2])) if reference is None: raise _Unsupported("orphan reflog lookup") cursor = reference.log_iter().rev() @@ -1174,7 +1175,7 @@ def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: if delete: if len(operands) != 1 or deref: raise _Unsupported("reference deletion options") - edit = gix.RefEdit.delete(operands[0], gix.PreviousValue.Any).with_deref(False) + edit = gix.RefEdit.delete(safe_encode(operands[0]), gix.PreviousValue.Any).with_deref(False) _write("update_ref", lambda: repo.edit_references_as([edit])) return b"" if len(operands) != 2: @@ -1184,7 +1185,8 @@ def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: kind = repo.find_header(oid).kind() if (name == "HEAD" or name.startswith("refs/heads/")) and kind != "commit": raise _Unsupported("branch target validation") - reference = repo.try_find_reference(name) + encoded_name = safe_encode(name) + reference = repo.try_find_reference(encoded_name) if reference is not None: if deref and name != "HEAD" and reference.target().try_name() is not None: raise _Unsupported("symbolic reference update (GIX-10)") @@ -1196,13 +1198,15 @@ def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes: for _ in range(5): if head is None or head.target().try_name() is None: break - if head.target().try_name() == os.fsencode(name): + if head.target().try_name() == encoded_name: raise _Unsupported("active branch HEAD reflog (GIX-10)") head = head.follow() log = gix.LogChange() log.force_create_reflog = force_log log.message = message - edit = gix.RefEdit.update_with_log(name, gix.Target.Object(oid), gix.PreviousValue.Any, log).with_deref(deref) + edit = gix.RefEdit.update_with_log(encoded_name, gix.Target.Object(oid), gix.PreviousValue.Any, log).with_deref( + deref + ) signature = _committer(repo, kwargs.get("env", {})) _write("update_ref", lambda: repo.edit_references_as([edit], signature)) return b"" diff --git a/git/compat.py b/git/compat.py index 2fb1c8474..cc77f206a 100644 --- a/git/compat.py +++ b/git/compat.py @@ -65,9 +65,9 @@ def safe_encode(s: AnyStr) -> bytes: ... def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: - """Safely encode a binary string to Unicode.""" + """Encode text to bytes, preserving bytes recovered by :func:`safe_decode`.""" if isinstance(s, str): - return s.encode(defenc) + return s.encode(defenc, "surrogateescape") elif isinstance(s, bytes): return s elif s is None: diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 197ca333c..d7d811fb3 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -11,6 +11,7 @@ from git import _backend from git.cmd import Git +from git.compat import safe_encode from git.exc import GitCommandError, UnsafeOptionError from git.objects.base import Object from git.objects.commit import Commit @@ -462,7 +463,7 @@ def _transaction(repo: "Repo", commands: str, logmsg: Union[str, None] = None) - if first_line: options.extend(("-m", first_line)) with tempfile.TemporaryFile() as stream: - stream.write(commands.encode("utf-8")) + stream.write(safe_encode(commands)) stream.seek(0) repo.git._call_process_safe("update_ref", "--no-deref", "--stdin", "-z", *options, istream=stream) diff --git a/test/test_refs.py b/test/test_refs.py index 1ed416ebd..af130d3c0 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -10,6 +10,8 @@ from pathlib import Path from unittest.mock import patch +import pytest + from gitdb.exc import BadName import git.refs as refs @@ -17,13 +19,16 @@ Commit, GitCommandError, GitConfigParser, + Git, Head, Reference, RefLog, RemoteReference, SymbolicReference, TagReference, + _backend, ) +from git.compat import safe_encode from git.exc import UnsafeOptionError from git.objects.tag import TagObject from git.util import Actor @@ -31,6 +36,27 @@ from test.lib.fixtures import fixture_repo, read_only_fixture, writable_fixture +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_non_utf8_tag_names_preserve_bytes_and_resolve(object_format): + with fixture_repo("legacy-refs.sh", object_format) as repo: + tags = repo.tags + assert len(tags) == 2 + assert {safe_encode(tag.path) for tag in tags} == {b"refs/tags/release-\xf5", b"refs/tags/annotated-\xe3"} + native_guard = ( + patch.object(Git, "execute", side_effect=AssertionError("unexpected reference CLI call")) + if _backend.name == "gix" + else contextlib.nullcontext() + ) + with native_guard: + for tag in tags: + assert tag.commit == repo.head.commit + assert repo.rev_parse(tag.path).hexsha == tag.object.hexsha + assert repo.commit(tag.path) == repo.head.commit + if _backend.name == "gix": + with patch.object(_backend, "gix", None): + assert {tag.path for tag in repo.tags} == {tag.path for tag in tags} + + class TestRefs(TestBase): def test_from_path(self): # Should be able to create any reference directly. diff --git a/tests/fixtures/legacy-refs.sh b/tests/fixtures/legacy-refs.sh new file mode 100644 index 000000000..ad15f5ff8 --- /dev/null +++ b/tests/fixtures/legacy-refs.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main --object-format="$1" repo +cd repo +tree=$(git mktree 1100000000 +0000\n\nrelease\n' "$commit" | git hash-object -t tag -w --stdin) +# Packed refs can carry bytes that the fixture filesystem rejects in loose names. +printf '# pack-refs with: peeled fully-peeled sorted\n%s refs/tags/annotated-\343\n^%s\n%s refs/tags/release-\365\n' \ + "$tag" "$commit" "$commit" >.git/packed-refs From 390fe8b5bbea0f67c98c7c16df9b9a1a6071a709 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 09:00:31 +0100 Subject: [PATCH 05/21] docs(repo): demonstrate closing repositories before deletion (#2101) Reading `repo.head.commit` can open persistent object readers or retain a native repository handle. Deleting the repository while its handles remain open is not a supported sequence on Windows. Git's read-only object files can independently prevent plain `shutil.rmtree()` from removing a clone. `Repo.close()` and its context-manager support already release the retained handle and terminate persistent Git readers synchronously. Document that contract in the API reference, tutorial and README, replacing the old advice to invoke destructors manually. Demonstrate closing the repository first, then using the existing strict `git.util.rmtree()` helper for read-only files. All instances and separately opened readers must be closed by their owners. Add a packed-repository fixture and a tutorial regression that reads HEAD and blob data, verifies reader processes have exited and caches are released, and removes the directory without suppressing deletion errors. The test passes with both CLI and Gix backends on macOS with Git 2.54.0. Sphinx's strict HTML build, Ruff and ShellCheck pass. Windows execution is left to CI; no automatic cleanup of a still-live repository is introduced. --- README.md | 8 ++++++-- doc/source/tutorial.rst | 26 ++++++++++++++++++++++++++ git/repo/base.py | 7 +++++++ test/test_docs.py | 23 +++++++++++++++++++++++ tests/fixtures/repo-close.sh | 7 +++++++ 5 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/repo-close.sh diff --git a/README.md b/README.md index da8e7c894..6f740f7d9 100644 --- a/README.md +++ b/README.md @@ -161,8 +161,12 @@ GitPython is not suited for long-running processes (like daemons) as it tends to leak system resources. It was written in a time where destructors (as implemented in the `__del__` method) still ran deterministically. -In case you still want to use it in such a context, you will want to search the -codebase for `__del__` implementations and call these yourself when you see fit. +Use `Repo` as a context manager (`with Repo(path) as repo:`), or call +`repo.close()` explicitly when finished, to release its native repository handle +and persistent Git object readers. Close every repository instance and any +separately opened readers before removing its directory. On Windows, +`git.util.rmtree(path)` also handles Git's read-only files, which plain +`shutil.rmtree()` may be unable to remove even after handles are closed. Another way assure proper cleanup of resources is to factor out GitPython into a separate process which can be dropped periodically. diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 29c9c4e31..a56d2ac0c 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -63,6 +63,32 @@ Archive the repository contents to a tar file. :start-after: # [6-test_init_repo_object] :end-before: # ![6-test_init_repo_object] +Closing a repository +==================== + +Use a :class:`~git.repo.base.Repo` context manager, or call +:meth:`~git.repo.base.Repo.close` explicitly when finished. Exiting the context +releases its native handle and persistent Git object readers, including when an +exception is raised. + +.. literalinclude:: ../../test/test_docs.py + :language: python + :dedent: 8 + :start-after: # [close-repo-context] + :end-before: # ![close-repo-context] + +Close every repository instance and any separately opened readers before deleting +the directory. Windows prevents deletion while files remain open; Git's read-only +object files can also make plain ``shutil.rmtree()`` fail. The existing +:func:`git.util.rmtree` helper handles those read-only files while still reporting +deletion failures: + +.. literalinclude:: ../../test/test_docs.py + :language: python + :dedent: 8 + :start-after: # [remove-repo-directory] + :end-before: # ![remove-repo-directory] + Advanced Repo Usage =================== diff --git a/git/repo/base.py b/git/repo/base.py index 0dd0206dc..f03a52b44 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -535,6 +535,13 @@ def __del__(self) -> None: pass def close(self) -> None: + """Release native repository state and persistent Git object readers. + + Use ``with Repo(path) as repo:`` to close automatically, including on errors. + Close every repository instance and any separately opened readers before + removing its directory, especially on Windows where open handles prevent + deletion. :func:`git.util.rmtree` also handles Git's read-only files. + """ self._gix_repository = self._gix_state = None if self.git: self.git.clear_cache() diff --git a/test/test_docs.py b/test/test_docs.py index 5594c9423..97166e1ea 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -12,6 +12,29 @@ from test.lib.fixtures import writable_fixture +def test_repo_context_releases_resources_before_removal(): + with writable_fixture("repo-close.sh") as temporary: + path = temporary.path() / "repo" + # [close-repo-context] + from git import Repo + + with Repo(path) as repo: + print(repo.head.commit.hexsha) + # ![close-repo-context] + assert repo.head.commit.tree["file"].data_stream.read() == b"contents\n" + processes = [reader.proc for reader in (repo.git.cat_file_header, repo.git.cat_file_all) if reader] + + assert repo._gix_repository is None + assert repo.git.cat_file_header is repo.git.cat_file_all is None + assert all(process.poll() is not None for process in processes) + # [remove-repo-directory] + from git.util import rmtree + + rmtree(path) + # ![remove-repo-directory] + assert not path.exists() + + class Tutorials(TestBase): def tearDown(self): gc.collect() diff --git a/tests/fixtures/repo-close.sh b/tests/fixtures/repo-close.sh new file mode 100644 index 000000000..6a7c7af27 --- /dev/null +++ b/tests/fixtures/repo-close.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main repo +printf 'contents\n' >repo/file +git -C repo add file +git -C repo commit -m initial +git -C repo gc From 932032bb931e1a5b1c8e75f65cd7c83791940753 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 09:44:10 +0100 Subject: [PATCH 06/21] test: respect Windows argument encoding in packed-tag regressions The Windows Python 3.15 free-threaded CI job enumerated the non-UTF-8 packed tags correctly, then failed both object-format variants when the new regression tried to resolve their surrogate-escaped names through the CLI. Git 2.56.0.windows.2 returned exit 128 from `git rev-parse --verify --end-of-options` with `fatal: Needed a single revision`. Git's Windows `wmain()` converts UTF-16 arguments to UTF-8, so those arguments cannot carry the original invalid UTF-8 reference bytes. Separate packed-tag enumeration from reference resolution. Keep the reported enumeration regression on both backends and every platform; skip only the Windows CLI resolution variant. Gix resolution still runs on every platform with `Git.execute()` forbidden, using the supported byte-capable native APIs. Document the demonstrated CLI limitation and the coverage boundary in `doc/gix-backend.md`. Validation: all 44 reference tests and 24 subtests pass locally on each backend; Ruff check/format and `git diff --check` pass. Windows execution remains for CI. --- doc/gix-backend.md | 15 +++++++++++---- test/test_refs.py | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/doc/gix-backend.md b/doc/gix-backend.md index 6e44ab852..86e781816 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -393,10 +393,17 @@ Reference names and revisions are encoded with `surrogateescape` before calling GixPython's byte-capable `Target.Symbolic`, `Repository.try_find_reference`, `Repository.rev_parse_single`, `Repository.rev_parse` and `RefEdit` APIs. This preserves non-UTF-8 packed tag names without interpreting reference storage -in Python. `test_non_utf8_tag_names_preserve_bytes_and_resolve` covers lightweight -and annotated tags in SHA-1 and SHA-256 repositories and forbids CLI fallback -during native resolution. This corrects adapter input conversion; GixPython 0.3.0 -already accepts the original bytes through those APIs. +in Python. `test_non_utf8_tag_names_preserve_bytes` covers lightweight and annotated +tag enumeration in SHA-1 and SHA-256 repositories on both backends and all platforms. +`test_non_utf8_tag_names_resolve` forbids CLI fallback during native resolution. +Its CLI variant requires POSIX: Windows command-line arguments cannot preserve +the original non-UTF-8 bytes. Windows CI with Git 2.56.0.windows.2 and CPython +3.15.1 free-threaded enumerates both packed names, but resolving the +surrogate-escaped `annotated-\udce3` name through `git rev-parse --verify` exits 128 +with `fatal: Needed a single revision`. Enumeration remains tested on Windows +CLI; Gix resolution remains tested on Windows through the byte-capable APIs. +This corrects adapter input conversion; GixPython 0.3.0 already accepts the +original bytes through those APIs. Repository-backed native operations fall back for reftable (GIX-1) and repositories with `extensions.compatObjectFormat` (GIX-19). Process-returning calls, timeouts, diff --git a/test/test_refs.py b/test/test_refs.py index af130d3c0..48a285b4f 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -4,6 +4,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import contextlib +import os import os.path as osp import unittest from itertools import chain @@ -37,11 +38,23 @@ @pytest.mark.parametrize("object_format", ["sha1", "sha256"]) -def test_non_utf8_tag_names_preserve_bytes_and_resolve(object_format): +def test_non_utf8_tag_names_preserve_bytes(object_format): with fixture_repo("legacy-refs.sh", object_format) as repo: tags = repo.tags assert len(tags) == 2 assert {safe_encode(tag.path) for tag in tags} == {b"refs/tags/release-\xf5", b"refs/tags/annotated-\xe3"} + if _backend.name == "gix": + with patch.object(_backend, "gix", None): + assert {tag.path for tag in repo.tags} == {tag.path for tag in tags} + + +@pytest.mark.skipif( + os.name == "nt" and _backend.name == "cli", reason="Windows command-line arguments cannot preserve non-UTF-8 bytes" +) +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_non_utf8_tag_names_resolve(object_format): + with fixture_repo("legacy-refs.sh", object_format) as repo: + tags = repo.tags native_guard = ( patch.object(Git, "execute", side_effect=AssertionError("unexpected reference CLI call")) if _backend.name == "gix" @@ -52,9 +65,6 @@ def test_non_utf8_tag_names_preserve_bytes_and_resolve(object_format): assert tag.commit == repo.head.commit assert repo.rev_parse(tag.path).hexsha == tag.object.hexsha assert repo.commit(tag.path) == repo.head.commit - if _backend.name == "gix": - with patch.object(_backend, "gix", None): - assert {tag.path for tag in repo.tags} == {tag.path for tag in tags} class TestRefs(TestBase): From 098db6b085ee61ddbf7f9e85af31f20f40198f1a Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 09:54:28 +0100 Subject: [PATCH 07/21] test(gix): record Windows byte-reference lookup fallback The Windows Gix job also failed both packed-tag resolution regressions: enumeration preserved the names, but `reference_info()` selected its native read diagnostic fallback before the no-CLI guard caught a Git launch. The run used GixPython 0.3.0, Git 2.56.0.windows.2 and CPython 3.12.10. The pinned `gix-ref` 0.69.1 lookup attempts loose-reference path conversion before consulting packed refs, which cannot represent invalid UTF-8 bytes on Windows. The CI log demonstrates the fallback, but does not contain the underlying Gix diagnostic text. Keep byte-name resolution coverage on POSIX and check the existing native fallback explicitly on Windows. Packed-tag enumeration remains covered on both backends on every platform, resolving the reported enumeration problem without claiming unsupported native lookup coverage. Record the open GIX-32 capability gap and required upstream lookup support in `doc/gix-backend.md`. Git's Windows CLI also cannot address those original bytes through UTF-16 arguments, so this evidence is not classified as a demonstrated Git compatibility mismatch. No library behavior changes. Validation: all four focused packed-tag regressions pass on each local backend; Ruff check/format and `git diff --check` pass. The Windows fallback assertion remains for CI. --- doc/gix-backend.md | 12 +++++++----- test/test_refs.py | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/gix-backend.md b/doc/gix-backend.md index 86e781816..f768cc690 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -396,14 +396,15 @@ This preserves non-UTF-8 packed tag names without interpreting reference storage in Python. `test_non_utf8_tag_names_preserve_bytes` covers lightweight and annotated tag enumeration in SHA-1 and SHA-256 repositories on both backends and all platforms. `test_non_utf8_tag_names_resolve` forbids CLI fallback during native resolution. -Its CLI variant requires POSIX: Windows command-line arguments cannot preserve -the original non-UTF-8 bytes. Windows CI with Git 2.56.0.windows.2 and CPython +Resolution requires POSIX: Windows command-line arguments cannot preserve +the original non-UTF-8 bytes, and native Windows lookup has GIX-32. Windows CI +with Git 2.56.0.windows.2 and CPython 3.15.1 free-threaded enumerates both packed names, but resolving the surrogate-escaped `annotated-\udce3` name through `git rev-parse --verify` exits 128 with `fatal: Needed a single revision`. Enumeration remains tested on Windows -CLI; Gix resolution remains tested on Windows through the byte-capable APIs. -This corrects adapter input conversion; GixPython 0.3.0 already accepts the -original bytes through those APIs. +on both backends, along with the GIX-32 native diagnostic fallback. This +corrects adapter input conversion; GixPython 0.3.0 accepts the original bytes +through those APIs on POSIX. Repository-backed native operations fall back for reftable (GIX-1) and repositories with `extensions.compatObjectFormat` (GIX-19). Process-returning calls, timeouts, @@ -949,6 +950,7 @@ identify the historical fixture where needed. | GIX-29 | Missing API | GixPython 0.2.0 exposes raw identity/message/signature bytes without a Git-compatible text reencoder. Git 2.54.0 recodes the complete commit or retains all bytes when conversion fails; per-field Python conversion differs. The Latin-1, unknown/non-text codecs, malformed UTF-16 and signed-header cases in `test_deserialization_declared_encoding` reproduce the required behavior. | Expose whole-commit text conversion, including failed/unknown-encoding behavior. `object_metadata()` retains CLI for non-UTF-8 declarations. The fallback obtains identities/message from Git's fixed UTF-8 text projection and signatures from its recoded raw header block, retaining the original declaration separately. | | GIX-30 | Bug (open) | GixPython 0.2.0 `Commit.decode()` reports `encoding=None` when `encoding ISO-8859-1` follows `x-extra value`; Git 2.54.0 selects `ISO-8859-1` and renders Latin-1 identity/message bytes correctly. Gix retains `(b"encoding", b"ISO-8859-1")` in `extra_headers`, but its encoding field disagrees with Git. The `late-latin1` read-only `commit.sh` fixture in `test_deserialization_declared_encoding` reproduces this and asserts the fallback. | Recognize encoding headers in Git-compatible order. `object_metadata()` conservatively retains CLI when an encoding appears in `extra_headers`; it does not select the header in Python. Keep this bug open until an upstream fix or compatible mode is verified. | | GIX-31 | Bug (open) | GixPython 0.2.0 `Commit.decode()` rejects commit headers containing a CR-only or tab-only line with `object parsing failed, at gix-object/src/commit/decode.rs:20`. Git 2.54.0 accepts both objects with `fsck --strict`, continues through their header block and renders the later Latin-1 declaration and message correctly. A space-only continuation line decodes natively but retains the late encoding declaration in `extra_headers` (GIX-30). `test_deserialization_headers_end_at_empty_line` reproduces all three using the `commit.sh whitespace-headers` fixtures and checks Git's projection. | Accept Git-compatible header lines and end headers only at an empty line. The existing `object_metadata()` native decoder diagnostics fallback retains Git for CR/tab inputs; the existing GIX-30 fallback covers the space input. Neither fallback closes these bugs. | +| GIX-32 | Missing capability (open); no demonstrated Git compatibility mismatch | Windows CI on GixPython 0.3.0, Git 2.56.0.windows.2 and CPython 3.12.10 enumerates the non-UTF-8 packed names in `legacy-refs.sh`, but exact byte-name lookup selects `Reference.read: CLI: native read diagnostics`; the no-CLI guard fails for SHA-1 and SHA-256. POSIX lookup succeeds. Git's Windows CLI likewise cannot address those original bytes through UTF-16 arguments, so there is no demonstrated equivalent Windows CLI operand. Git enumeration does preserve both names. | Provide native Windows packed-reference lookup for byte names without first converting the name to a loose-reference filesystem path. Keep `reference_info()`'s existing diagnostic fallback; `test_non_utf8_tag_names_preserve_bytes` checks it on Windows. Resolution coverage and native claims remain POSIX-only. The upstream lookup source attempts loose-path conversion before consulting packed refs; CI demonstrates the fallback, but does not capture the underlying Gix error text. | The PR #2306 regression exposed an adapter ownership-check bypass in both backends: `Repo`'s automatic `GIT_DIR` made Git 2.54.0 treat hook execution as diff --git a/test/test_refs.py b/test/test_refs.py index 48a285b4f..5007b29e8 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -44,12 +44,16 @@ def test_non_utf8_tag_names_preserve_bytes(object_format): assert len(tags) == 2 assert {safe_encode(tag.path) for tag in tags} == {b"refs/tags/release-\xf5", b"refs/tags/annotated-\xe3"} if _backend.name == "gix": + if os.name == "nt": + # GIX-32: byte-name lookup cannot reach packed refs on Windows. + for tag in tags: + assert _backend.reference_info(repo.git, tag.path) is NotImplemented with patch.object(_backend, "gix", None): assert {tag.path for tag in repo.tags} == {tag.path for tag in tags} @pytest.mark.skipif( - os.name == "nt" and _backend.name == "cli", reason="Windows command-line arguments cannot preserve non-UTF-8 bytes" + os.name == "nt", reason="Windows CLI arguments cannot preserve non-UTF-8 bytes; native lookup has GIX-32" ) @pytest.mark.parametrize("object_format", ["sha1", "sha256"]) def test_non_utf8_tag_names_resolve(object_format): From d697e507cb1d01b79fb24c2b27d45f2fd5e96c3a Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 11:02:05 +0100 Subject: [PATCH 08/21] fix(tag): address missing tagger review feedback Review item https://github.com/gitpython-developers/GitPython/pull/2274#discussion_r4368398028 reports that "reading them raises `AttributeError`" when native decoding has no tagger, and requests preserving initialized fields through a CLI fallback. Four SHA-1/SHA-256 regressions reproduce the native failure. The two cases with a message also expose CLI loss of the first message line, so falling back alone would not restore complete metadata. Adapt `Tag.tagger()` returning `None` to the existing empty `Actor` and zero date/offset defaults through `parse_actor_and_date("")`. Gix already exposes the absence correctly; no new Git semantics or fallback is required. Make the CLI parser recognize the optional tagger header and preserve the entire message. Git 2.54.0's `tag.c::parse_tag_buffer()` accepts an absent tagger and sets its date to zero. The existing GIX-28 strict writer gap remains open. All four regressions now pass on both backends; the native cases forbid `Git.execute()` during metadata access. Reference and CLI object/index tests pass on CLI (105 tests, 24 subtests) and Gix (105 tests, 24 subtests). Ruff, ShellCheck and mypy pass. --- doc/gix-backend.md | 4 +++- git/objects/tag.py | 23 +++++++---------------- test/test_refs.py | 21 +++++++++++++++++++++ tests/fixtures/refs.sh | 10 +++++++++- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/doc/gix-backend.md b/doc/gix-backend.md index f768cc690..aa4386fe0 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -130,7 +130,9 @@ these editing fallbacks. Commit metadata uses `Repository.find_commit().decode()` plus structured `Commit.author()`/`committer()` signatures. Tag metadata uses -`Repository.find_tag().decode()` and `Tag.tagger()`. Decoded extra headers retain +`Repository.find_tag().decode()` and `Tag.tagger()`. A missing native tagger maps +to GitPython's empty actor and zero date/offset; tag reads remain native. +Decoded extra headers retain `gpgsig`. Raw Gix object bytes are used only for object streams. If repository capabilities require CLI fallback, metadata uses `git cat-file`: general formatted Git queries cannot faithfully expose all signature headers or arbitrary tags. diff --git a/git/objects/tag.py b/git/objects/tag.py index cb2d63e5a..e0898c2fe 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -113,7 +113,9 @@ def _set_cache_(self, attr: str) -> None: self.object = object_type(self.repo, bytes.fromhex(data["target"])) self.tag = data["name"].decode(defenc, "replace") signature = data["tagger"] - if signature is not None: + if signature is None: + self.tagger, self.tagged_date, self.tagger_tz_offset = parse_actor_and_date("") + else: self.tagger = Actor( signature.name.decode(defenc, "replace"), signature.email.decode(defenc, "replace") ) @@ -144,21 +146,10 @@ def _set_cache_(self, attr: str) -> None: self.tag = lines[2][4:] # tag - if len(lines) > 3: - tagger_info = lines[3] # tagger - ( - self.tagger, - self.tagged_date, - self.tagger_tz_offset, - ) = parse_actor_and_date(tagger_info) - - # Line 4 empty - it could mark the beginning of the next header. - # In case there really is no message, it would not exist. - # Otherwise a newline separates header from message. - if len(lines) > 5: - self.message = "\n".join(lines[5:]) - else: - self.message = "" + has_tagger = len(lines) > 3 and lines[3].startswith("tagger ") + self.tagger, self.tagged_date, self.tagger_tz_offset = parse_actor_and_date(lines[3] if has_tagger else "") + message_start = 5 if has_tagger else 4 + self.message = "\n".join(lines[message_start:]) # END check our attributes else: super()._set_cache_(attr) diff --git a/test/test_refs.py b/test/test_refs.py index 5007b29e8..55fc7aa84 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -71,6 +71,27 @@ def test_non_utf8_tag_names_resolve(object_format): assert repo.commit(tag.path) == repo.head.commit +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +@pytest.mark.parametrize("message", ["", "release\nmore"]) +def test_tag_without_tagger_initializes_all_metadata(object_format, message): + with fixture_repo("refs.sh", "tag-no-tagger", object_format, "message" if message else "empty") as repo: + oid = (Path(repo.working_tree_dir).parent / "tag-id").read_text().strip() + expected = repo.head.commit + tag = TagObject(repo, bytes.fromhex(oid)) + native_guard = ( + patch.object(Git, "execute", side_effect=AssertionError("unexpected tag metadata CLI call")) + if _backend.name == "gix" + else contextlib.nullcontext() + ) + with native_guard: + assert tag.object == expected + assert tag.tag == "no-tagger" + assert tag.tagger == Actor("", None) + assert tag.tagged_date == 0 + assert tag.tagger_tz_offset == 0 + assert tag.message == message + + class TestRefs(TestBase): def test_from_path(self): # Should be able to create any reference directly. diff --git a/tests/fixtures/refs.sh b/tests/fixtures/refs.sh index 377181443..3c19a1d8a 100644 --- a/tests/fixtures/refs.sh +++ b/tests/fixtures/refs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -eu -git init --initial-branch=master repo +git init --initial-branch=master --object-format="${2:-sha1}" repo cd repo printf 'initial\n' > file.txt git add file.txt @@ -22,6 +22,14 @@ case "$1" in printf 'tagger Mallory 1700000001 +0000\n\nrelease\n' } | git hash-object -w -t tag --literally --stdin >../tag-id ;; + tag-no-tagger) + { + printf 'object %s\ntype commit\ntag no-tagger\n\n' "$(git rev-parse HEAD)" + if test "$3" = message; then + printf 'release\nmore\n' + fi + } | git hash-object -w -t tag --literally --stdin >../tag-id + ;; outside) git rev-parse HEAD > ../outside-ref ;; From 348ebbec2bca67dd2a53d83a3148e7e628cb9b2b Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 11:02:36 +0100 Subject: [PATCH 09/21] fix(tree): address partial cache review feedback Review item https://github.com/gitpython-developers/GitPython/pull/2274#discussion_r4368398055 reports that "subsequent iteration or lookup silently accepts that truncated tree" after an entry fails validation. Stored trees containing a `.git` entry reproduce this on both backends: the first iteration rejects the name, but the second accepts an empty tree or the valid prefix. Build the entries in a local list and attach `_cache` only after every record passes parsing and validation. Failed reads leave the lazy cache unset, so later accesses retry and raise the same error instead of exposing partial data. No backend behavior or tree validation policy changes. Git 2.54.0's `mktree` and `ls-tree` accept and emit the fixture's `.git` entry; GitPython applies its existing metadata-path guard while consuming that output. Four regressions cover invalid entries first and last in SHA-1/SHA-256 trees. All fail before the fix and pass afterwards on CLI and native Gix `ls_tree`. All 97 tree tests pass on each backend. Ruff, ShellCheck and mypy pass. --- git/objects/tree.py | 5 +++-- test/test_tree.py | 14 ++++++++++++++ tests/fixtures/tree-invalid.sh | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/tree-invalid.sh diff --git a/git/objects/tree.py b/git/objects/tree.py index f04e0d75c..f9361558e 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -219,7 +219,7 @@ def _set_cache_(self, attr: str) -> None: output = self.repo.git._call_process_safe( "ls_tree", "-z", "--full-tree", self.hexsha, stdout_as_string=False ) - self._cache: List[TreeCacheTup] = [] + cache: List[TreeCacheTup] = [] for record in output.split(b"\0"): if not record: continue @@ -230,7 +230,8 @@ def _set_cache_(self, attr: str) -> None: binsha = bytes.fromhex(oid.decode("ascii")) if len(binsha) != self.repo._oid_size: raise ValueError("Invalid tree object ID returned by Git") - self._cache.append((binsha, int(mode, 8), name)) + cache.append((binsha, int(mode, 8), name)) + self._cache = cache else: super()._set_cache_(attr) # END handle attribute diff --git a/test/test_tree.py b/test/test_tree.py index 5c46027ec..3cc2de776 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -41,6 +41,20 @@ def test_tree_cache_edits_do_not_update_object_ids_or_parents(): assert repo.head.commit.tree["sub"].hexsha == original_oid +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +@pytest.mark.parametrize("invalid_position", ["first", "last"]) +def test_invalid_tree_does_not_retain_partial_cache(object_format, invalid_position): + with fixture_repo("tree-invalid.sh", object_format, invalid_position) as repo: + root = read_only_fixture("tree-invalid.sh", object_format, invalid_position) + oid = (root / "tree-id").read_text().strip() + names = [line.split("\t")[1] for line in (root / "entries").read_text().splitlines()] + assert names == ([".git", "valid"] if invalid_position == "first" else ["!valid", ".git"]) + tree = Tree(repo, bytes.fromhex(oid), path="") + for _ in range(2): + with pytest.raises(ValueError, match="aliases Git metadata"): + list(tree) + + @ddt.ddt class TestTree(TestBase): @with_rw_repo("HEAD") diff --git a/tests/fixtures/tree-invalid.sh b/tests/fixtures/tree-invalid.sh new file mode 100644 index 000000000..ea11f7c34 --- /dev/null +++ b/tests/fixtures/tree-invalid.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main --object-format="$1" repo +cd repo +blob=$(printf 'content\n' | git hash-object -w --stdin) +if test "$2" = first; then + valid=valid +else + valid='!valid' +fi +{ + printf '100644 blob %s\t.git\n' "$blob" + printf '100644 blob %s\t%s\n' "$blob" "$valid" +} | git mktree > ../tree-id +git ls-tree "$(cat ../tree-id)" > ../entries From 4dc547c91e1e07b104d6b10be83d79c546dac3de Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 11:04:06 +0100 Subject: [PATCH 10/21] fix(refs): address literal branch rename review feedback Review item https://github.com/gitpython-developers/GitPython/pull/2274#discussion_r4368398073 reports that "the returned head points to `refs/heads/topic`, which may be missing or identify a different branch" after renaming to the short branch name `refs/heads/topic`. A repository with an existing `topic` at commit A and a source branch at commit B reproduces the incorrect returned ref on both backends. Compute the destination by prepending `refs/heads/` to the literal name passed to `git branch`. Validate that same destination and retain it only after Git successfully renames the branch. Git 2.54.0's `builtin/branch.c::copy_or_rename_branch()` uses `refs.c::check_branch_ref()`, which always prepends that namespace. The general `to_full_path()` helper continues to normalize already-qualified references for its other callers. The SHA-1/SHA-256 regressions fail before this fix and pass afterwards. They check the returned path, name and commit, current `HEAD`, and that the original `topic` still points to commit A. All 50 reference tests and 24 subtests pass on each backend, including ordinary and forced renames. Ruff, ShellCheck, mypy and basedpyright pass. --- git/refs/head.py | 5 +++-- test/test_refs.py | 17 +++++++++++++++++ tests/fixtures/refs.sh | 5 +++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/git/refs/head.py b/git/refs/head.py index 889661588..4f348080d 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -252,10 +252,11 @@ def rename(self, new_path: PathLike, force: bool = False) -> "Head": flag = "-M" new_name = Git._check_operand(new_path, "branch") + new_full_path = "%s/%s" % (self._common_path_default, new_name) self._get_validated_ref_path(self.repo, self.path) - self._get_validated_ref_path(self.repo, self.to_full_path(new_name)) + self._get_validated_ref_path(self.repo, new_full_path) self.repo.git._call_process_safe("branch", flag, "--", self.name, new_name) - self.path = str(self.to_full_path(new_name)) + self.path = new_full_path return self def checkout( diff --git a/test/test_refs.py b/test/test_refs.py index 55fc7aa84..b47fe40e5 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -92,6 +92,23 @@ def test_tag_without_tagger_initializes_all_metadata(object_format, message): assert tag.message == message +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_head_rename_retains_literal_prefixed_branch_name(object_format): + with fixture_repo("refs.sh", "rename-prefixed", object_format, write=True) as repo: + original_topic = repo.heads.topic.commit + source = repo.head.reference + source_commit = source.commit + assert source_commit != original_topic + renamed = source.rename("refs/heads/topic") + assert renamed is source + assert renamed.path == "refs/heads/refs/heads/topic" + assert renamed.name == "refs/heads/topic" + assert renamed.commit == source_commit + assert repo.head.reference == renamed + assert repo.head.commit == source_commit + assert repo.heads.topic.commit == original_topic + + class TestRefs(TestBase): def test_from_path(self): # Should be able to create any reference directly. diff --git a/tests/fixtures/refs.sh b/tests/fixtures/refs.sh index 3c19a1d8a..cc13ed00d 100644 --- a/tests/fixtures/refs.sh +++ b/tests/fixtures/refs.sh @@ -30,6 +30,11 @@ case "$1" in fi } | git hash-object -w -t tag --literally --stdin >../tag-id ;; + rename-prefixed) + git branch topic + printf 'second\n' >> file.txt + git commit -am second + ;; outside) git rev-parse HEAD > ../outside-ref ;; From 3b1eb071b59e9ba7f6f5a4775737080b9f9714f3 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 16:27:51 +0100 Subject: [PATCH 11/21] docs: align documentation with released behavior The documentation described temporary development regressions as restored public behavior, including a `TreeModifier` deprecation that never shipped. Audit current guidance and the changelog against `3.2.1` and the relevant historical release tags so dependants see the net changes from releases. Describe tree editing, configuration assignment, revision paths, literal paths and compatibility imports directly. Clarify that `GitDB` selects the managed backend, and scope the version lookup improvement to source checkouts. Remove the historical tree-removal entry: its announcement shipped in `1.0.2`, but the API was present in both `1.0.1` and `1.0.2`. Keep genuine released deprecations and fixes. Identify the Gix integration as unreleased and its measurements as development validation. Remove internal restoration and commit-rewrite narration while preserving compatibility bugs, reproduction evidence and measurement caveats. The strict Sphinx HTML build, Ruff lint/format checks and `git diff --check` pass. Python AST comparison confirms that source edits affect only docstrings and comments; runtime tests were not rerun for this documentation-only change. --- doc/gix-backend.md | 63 +++++++++++++++++++-------------------- doc/source/changes.rst | 64 +++++++++++++++++----------------------- doc/source/intro.rst | 2 +- doc/source/tutorial.rst | 7 +++-- fuzzing/README.md | 4 +-- git/db.py | 3 +- git/objects/tree.py | 7 ++--- git/repo/base.py | 15 +++++----- test/downstream/audit.md | 44 ++++++++++++++------------- 9 files changed, 100 insertions(+), 109 deletions(-) diff --git a/doc/gix-backend.md b/doc/gix-backend.md index aa4386fe0..74cd8556d 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -1,8 +1,8 @@ # GixPython backend -Install the `gix` extra to make GitPython use GixPython for supported operations: -`GitPython[gix]`, or `.[gix]` from this checkout. GixPython is the distribution -name; `gix` is its import name. The extra uses the official PyPI release +This backend is unreleased. Install `.[gix]` from this checkout to make GitPython +use GixPython for supported operations. GixPython is the distribution name; +`gix` is its import name. The extra uses the official PyPI release `GixPython==0.3.0` on CPython 3.11 or newer. On older Python versions, the dependency is omitted and GitPython uses the CLI backend. The ordinary GitPython installation still supports Python 3.8 or newer. @@ -14,7 +14,7 @@ uv venv --python .venv/bin/python .tox/gix uv pip install --python .tox/gix/bin/python --editable '.[test,gix]' ``` -`tox -e gix` also resolves the published release from the package index. +`tox -e gix` also resolves the published GixPython release from the package index. GixPython 0.3.0 publishes macOS wheels. On Windows and Linux, installation builds the released source distribution and requires Rust 1.89 or newer and @@ -39,7 +39,7 @@ selection also fails. With the variable unset, GitPython selects Gix when it can be imported and otherwise uses CLI. Test and benchmark runners select a backend explicitly, because both installations contain GixPython for fixtures. -Git 2.52 or newer remains required. Unsupported operations and call options use +Git 2.52 or newer is required. Unsupported operations and call options use the existing CLI implementation. Public `repo.git.()` calls retain their command-line behavior. Native dispatch applies to library-managed calls. @@ -55,7 +55,7 @@ fallbacks. ## Index and object metadata lifecycle -`Repo(..., expand_vars=False)` remains accepted for callers such as sbxloop +`Repo(..., expand_vars=False)` accepts literal paths for callers such as sbxloop 2.1.36 that explicitly require literal paths. The default also keeps environment variables literal; `expand_vars=True` is unsupported. Callers wanting expansion must perform it before passing the path. Discovery still uses Gix/Git after @@ -79,10 +79,11 @@ the old constructor. Downstream must construct its trusted metadata view before opening it with `Repo`, and handle native discovery errors explicitly; its original tests remain failed in the audit. -`git.GitDB` and `git.db.GitDB` remain compatibility aliases for +`git.GitDB` and `git.db.GitDB` are compatibility aliases for `GitCmdObjectDB`, so legacy `Repo(..., odbt=GitDB)` selectors use the same managed backend. Aider 0.86.2 uses that selector for every repository open. -This does not restore gitdb's Python object database or its specialized APIs. +These aliases select managed storage; gitdb's Python object database and its +specialized APIs are unsupported. Object queries continue through `object_data()` and the supported `Repository.try_find_header()`/`find_object()` APIs, with the existing large-object and diagnostic CLI fallbacks. Storage uses the existing `hash_object` dispatch. @@ -90,9 +91,9 @@ and diagnostic CLI fallbacks. Storage uses the existing `hash_object` dispatch. `Head.tracking_branch()` asks Git's `rev-parse --symbolic-full-name` to resolve local upstreams configured with `branch..remote=.`. GixPython does not expose the corresponding Git-compatible upstream configuration resolver; this -path retains CLI. Without it, both released GitPython 3.2.1 and the initial -adapter return an invalid remote reference, preventing angr 10.0.4.post1 from -reporting a valid local tracking branch. A native replacement must expose Git's +path retains CLI. Released GitPython 3.2.1 returns an invalid remote reference +for these upstreams, preventing angr 10.0.4.post1 from reporting a valid local +tracking branch. A native replacement must expose Git's local-upstream resolution, including its validation and diagnostics. Revision path results retain the root tree's empty path so `Repo.rev_parse("HEAD:")` @@ -136,7 +137,7 @@ Decoded extra headers retain `gpgsig`. Raw Gix object bytes are used only for object streams. If repository capabilities require CLI fallback, metadata uses `git cat-file`: general formatted Git queries cannot faithfully expose all signature headers or arbitrary tags. -The bounded public identity helpers remain Python glue. Human date parsing uses +The public identity helpers use Python glue. Human date parsing uses Git until GIX-27 is exposed; trailer extraction already uses Git. Benchmark remeasurement on 2026-10-09 used the same fixed SHA-1/files fixture @@ -346,11 +347,11 @@ include explicit CLI tests and fixture commands, including non-Git commands passed directly to `Git.execute`; they are a local baseline, not a count of only fallback calls. The expanded warm benchmark instead isolates a user journey: 43 CLI launches versus one Gix launch, opening nine versus one, and nested -discovery eleven versus three after restoring the reference-format fallback -and sharing successful version checks. -The historical full-suite counts above predate this correction. A constant -`files` answer was not a Gix query; `GIX-1` records the remaining native -repository-format validation gap. +discovery eleven versus three, including the reference-format fallback and +shared successful version checks. +The historical full-suite counts above include Python reference-format inference +and predate the current fallback; they do not establish native coverage. +`GIX-1` records the native repository-format validation gap. | GitPython entry point / managed command | Gix implementation | Remaining CLI cases | | --- | --- | --- | @@ -417,6 +418,9 @@ only an absent top-level `gix` selects CLI mode. ## Performance work +All measurements in this document describe unreleased GitPython development +revisions, using the GixPython and Git versions stated for each run. They are +validation evidence, not a history of changes between GitPython releases. Use the [measurements after the Gix-only audit](#measurements-after-the-gix-only-audit) for the corrected implementation. Older snapshots retain their original workloads and include shortcuts that were subsequently removed. @@ -534,9 +538,8 @@ this machine and workload rather than a general speed guarantee. The full Gix suite without coverage passed **1,678 tests and 38 subtests**, with 79 skips and one expected failure, in **337.51 seconds (5m 37.51s)**. -One expected path-expansion deprecation warning was reported. It recorded -**22,319 `Git.execute` launches**: 3,833 setup, 18,486 call, zero teardown or -collection/session. The process-lifetime count is 22,320 including the import +It recorded **22,319 `Git.execute` launches**: 3,833 setup, 18,486 call, zero +teardown or collection/session. The process-lifetime count is 22,320 including the import probe. Against the preceding 408.81-second/30,898-launch baseline, this local run took 17.4% less time and launched 27.8% fewer CLI processes; the suite has four additional regressions, and the timing comparison is observational. @@ -545,10 +548,7 @@ Affected tests ran with Gix before CLI: 304/264 repository, command, backend and safety tests; 136 command-guard tests per backend; 31 positional tests per backend; 13 process-count/budget tests per backend. Repository suites also passed 14 subtests, with three skips. Ruff lint/format, mypy and basedpyright -passed. Both code changes are separate Tix commits: `02e2cc44` retains native -handles and the original `c49bbec0` attempted zero-launch opening/discovery. -The rewritten opening commit restores real format queries and native metadata -fallbacks; the current coverage table and ceilings reflect those requirements. +passed. ### Measurements after the Gix-only audit @@ -563,7 +563,7 @@ The full-suite and timed benchmark figures in this section describe that snapshot, before the subsequent metadata and discovery changes. The full Gix suite without coverage passed **1,710 tests and 38 subtests**, -with 79 skips, one expected failure and one existing path-deprecation warning. +with 79 skips and one expected failure. It took **342.36 pytest seconds / 342.80 seconds including the runner (5m 42.80s)** on existing CPython 3.12.14/macOS arm64, official GixPython 0.1.0 and Git 2.54.0. This includes CLI-inventory and Trace2 overhead. @@ -647,8 +647,7 @@ fresh measurements and enforce the checked-in operation-specific ceilings. The follow-up used the same official GixPython 0.1.0 and existing CPython 3.12.14, with runtime and test code at -`ee947c98540396d5644f3a7fea7c807633f4b5b8`. Each implementation change was -amended into its corresponding Tix commit and checked with Gix before CLI. +`ee947c98540396d5644f3a7fea7c807633f4b5b8`, checked with Gix before CLI. - Reference storage can already be identified through native config. Direct SHA-1 and SHA-256 regressions confirm that reftable `HEAD` access raises @@ -943,7 +942,7 @@ identify the historical fixture where needed. | GIX-20 | Bug; missing API | `Target.Symbolic(name)` validates full names but rejects standalone names accepted by `git check-ref-format --allow-onelevel`. Four of 430 audited calls differed: `refs` twice, `hellothere`, and `valid_one_level_refname`. Further probes include digits, punctuation and Unicode (`1`, `A1`, `HEAD_1`, `A-B`, `A.B`, `Ä`). | Expose Git-compatible partial/one-level name validation, or correct the native restriction where appropriate. This is a binding/behavior shortcoming to revisit upstream; standalone names rejected natively retain CLI validation. | | GIX-21 | Bug; missing API | `Repository.committer()` preserves a process identity of ` Name. ` / ` email@example.invalid ` while `git var GIT_COMMITTER_IDENT` returns `Name.` / `email@example.invalid`. Native snapshot overrides for name, email and date work, but modifying the retained snapshot would leak per-command identity across threads. GixPython 0.1.0 has no general independent repository clone: Python copying fails and internal `RepoHandle::clone()` shares its `Arc` state. | Expose Git-compatible identity cleanup and a cheap independent native clone with isolated config, including per-command removal semantics. Reopening isolates state but repeats setup; `with_object_memory()` changes object-write behavior and is not a general clone. Keep `_checked_signature` validation and CLI fallback for cleanup/overrides. Never clean identity fields in Python or mutate shared snapshots/process environment to handle a command. | | GIX-22 | Missing API | The native Git directory is sufficient for the two fixed metadata leaves used here: `modules` and `COMMIT_EDITMSG`. Git's `path.c` applies no special relocation to either, including in linked worktrees. The adapter appends only these names to the canonical native Git directory. | No new binding is needed for these ordinary paths. Other `--git-path` names and symlinked metadata leaves retain CLI resolution; a general resolver must handle config/environment overrides, common/private storage and canonical targets. `Repository.modules_path()` means `.gitmodules`, not the `modules` storage directory. | -| GIX-23 | Missing API | No hook lookup or execution API is bound in GixPython 0.1.0. The removed absence fast path read config through Gix but used Python `os.stat()` to declare success. | Bind native hook lookup with configured/default path, linked-worktree, missing/nonexecutable-hook and diagnostic semantics, plus execution where needed. Until then all managed hook calls use Git, including `--ignore-missing` no-ops, preceded by a Git discovery/ownership check that removes only `Repo`'s automatic environment bindings. Each valid hook request therefore makes two CLI calls. Explicit caller overrides retain Git's behavior. A Python filesystem check is not a Gix implementation. | +| GIX-23 | Missing API | No hook lookup or execution API is bound in GixPython 0.1.0. | Bind native hook lookup with configured/default path, linked-worktree, missing/nonexecutable-hook and diagnostic semantics, plus execution where needed. Until then all managed hook calls use Git, including `--ignore-missing` no-ops, preceded by a Git discovery/ownership check that removes only `Repo`'s automatic environment bindings. Each valid hook request therefore makes two CLI calls. Explicit caller overrides retain Git's behavior. A Python filesystem check is not a Gix implementation. | | GIX-24 | Bug | In both object formats, `new_commit_as()` accepts a blob as the tree or a parent where `git commit-tree` rejects it. Tree-editor `upsert()`/`write()` accepts object IDs whose actual kinds disagree with blob/tree/gitlink modes; `git mktree --missing` rejects all three tested mismatches. `edit_references_as()` accepts a blob target under `refs/heads/`, rejected by `git update-ref`. | Provide checked commit/tree construction and reference edits, at least in Git-strict mode. Match Git's kind and direct-branch target validation before writing and under the required ref locks, including its distinct handling of symbolic aliases. Existing `_commit_tree`, `_write_tree` and `_update_ref` guards query Gix headers and select CLI on invalid inputs; they do not replace Gix writes with Python. | | GIX-25 | Bug; missing API | After `git symbolic-ref refs/heads/dangling refs/heads/missing`, `Repository.references().all()` enumerates the dangling name, while `git for-each-ref --format=%(refname)` omits it. Valid symbolic aliases must remain present. Both SHA-1 and SHA-256 probes reproduce this difference. | Expose Git-compatible enumeration with the same dangling-reference and diagnostic behavior, retaining the raw iterator for callers that need it. `_for_each_ref` forces Gix to resolve symbolic targets and falls back to Git on failure; keep that fallback until a compatible Gix API/mode is verified. | | GIX-26 | Bug; missing API | `dangerously_push_entry(IndexStat(), oid, 0, 0o120000, b".gitmodules")`, followed by `sort_entries()`, `verify_entries()` and `write()`, succeeds and retains the unsafe symlink. Protected Git `update-index -z --index-info` ignores it with exit 0 and `Ignoring path .gitmodules`; protected `read-tree` rejects it. The native fresh `index_from_tree` also rejects it. | A Git-strict checked index batch editor that validates names/modes, preserves metadata and reports rejected records. `verify_entries()` verifies ordering, not pathname safety. All private-index edits use Git with `core.protectHFS=true` and `core.protectNTFS=true`, followed by requested-entry readback; no Python dictionary reconstruction or native dangerous insertion remains. | @@ -1205,7 +1204,7 @@ or serialized in Python. GixPython needs a supported native configuration edit that creates sections before this contract can be restored. Kas's separate identity cleanup uses nonempty section assignment, which is -restored through existing Git config remove/set operations under GIX-12. Values +supported through Git config remove/set operations under GIX-12. Values are validated before mutation; individual Git operations lock independently, so replacement does not claim transaction-wide atomicity. Downstream can avoid the empty-section API by calling `set_value('gpg "ssh"', 'allowedSignersFile', @@ -1268,8 +1267,8 @@ The full baseline at `f6779dbb` reproduced all 17 failures described above: 16 ordinary failures and the subtest with malformed `commondir`. The full patched suite passed with **1,760 tests and 40 subtests**, 58 skipped, nine expected failures and three unexpected passes in 2,242.66 seconds. The three -unexpected passes and the path-deprecation warning also appeared in the -baseline. Ruff, mypy, basedpyright and the pinned pre-commit checks passed. +unexpected passes also appeared in the baseline. Ruff, mypy, basedpyright and +the pinned pre-commit checks passed. The baseline recorded 21,598 CLI launches in 2,279.10 seconds; the patched run recorded 21,826 CLI launches and 86,936 native operations. Totals include the additional @@ -1285,7 +1284,7 @@ reference and mutation-error cases also passed under Ubuntu WSL: 43 tests and 14 subtests, using CPython 3.14.4, Git 2.53.0 and a separate released-source GixPython build. This includes the POSIX newline-filename cases. -## Published release validation +## GixPython 0.1.0 artifact validation The official Apple Silicon ABI3 wheel for GixPython 0.1.0 has SHA-256 `6b68758cb54a90c8eb893ac7815e74ccd9b48b197aaecec951366a6326168f52`. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 65f03c548..9b2aaf3f0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -51,40 +51,28 @@ Git executable and safety API changes ----------- -* Packed tag names containing non-UTF-8 bytes remain losslessly represented as +* Packed tag names containing non-UTF-8 bytes are represented losslessly as surrogate-escaped strings. ``safe_encode()`` recovers their original bytes, and Gix reference validation and resolution receive those bytes directly. * Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``, and raw rename fields. Staged-file diffs no longer fail while decoding legacy filenames; blob contents remain byte streams regardless of their encoding. -* ``Tree.cache`` and ``TreeModifier`` remain supported. They edit one tree's - in-memory entries; object IDs and parent trees do not change automatically. - Tree serialization now delegates construction to Git or the supported Gix - backend. Prefer ``IndexFile.write_tree()`` for changes to staged paths. +* Tree serialization now delegates construction to Git or the supported Gix + backend. ``Tree.cache`` and ``TreeModifier`` edit one tree's in-memory entries; + object IDs and parent trees do not change automatically. Prefer + ``IndexFile.write_tree()`` for changes to staged paths. * ``Commit.stats`` preserves Unicode line separators in filenames when ``core.quotePath=false``. Git's LF-delimited output remains intact for root, - subsequent and empty commits, including the Gix backend's existing filename + subsequent and empty commits, including the Gix backend's filename quoting fallback. -* ``GitConfigParser[section] = values`` again replaces a nonempty section, - restoring Kas's Git identity cleanup. Git performs each mutation and locks - it independently; replacement is not one atomic transaction. Empty sections - and ``add_section()`` remain unsupported, including Kas's SSH key setup. -* Explicit ``from git import Union`` and ``Optional`` imports remain available - for Demisto SDK's validator registration and MkDocs RSS Plugin imports, - retaining the historical ``typing`` aliases. -* Editable/source imports expose the release version in ``git.__version__``, - matching built distributions. Salt's GitPython provider checks can parse the - version instead of rejecting the unstamped ``"git"`` build placeholder. +* Imports from a source checkout now read ``git.__version__`` from ``VERSION``. + Built wheels and source distributions continue to carry their stamped release + version. * ``Repo`` construction validates repository configuration through Git/Gix. A subclass override of ``config_reader()`` no longer controls discovery or suppresses its include graph. Malformed configuration and invalid repository - extensions can fail before downstream custom validation runs. sbxloop's - metadata-reader subclass exposes this remaining compatibility break; callers - must prepare their trusted metadata view before opening it with ``Repo``. -* Revision root trees such as ``Repo.rev_parse("HEAD:")`` retain their empty - path and can be iterated. Missing paths within a valid tree revision retain - ``Tree.join()``'s ``KeyError`` and missing-component message, restoring MCP - Server Git's directory display and path-error reporting. + extensions can fail before downstream custom validation runs. Callers using + custom metadata views must prepare them before opening with ``Repo``. * ``Head.tracking_branch()`` returns a local ``Head`` when Git configures the current repository (``branch..remote=.``) as the upstream, instead of constructing an invalid ``refs/remotes/./...`` reference. Git resolves the @@ -133,16 +121,21 @@ API changes * ``GitConfigParser`` uses Git's configuration syntax and canonical key spelling, and no longer subclasses ``configparser.RawConfigParser``. Common getters, typed values, duplicate values, file/stream inputs and mutation - methods remain. Invalid Git syntax previously accepted by Python is rejected. - Empty-section creation and valueless writes are removed; valueless reads remain. - Relative includes from streams are rejected by Git. Locks cover each native - mutation rather than the writer object's lifetime. + methods remain, including ``GitConfigParser[section] = values`` for replacing + nonempty sections. Invalid Git syntax previously accepted by Python is rejected. + Empty-section creation, including ``add_section()``, and valueless writes are + removed; valueless reads remain. Relative includes from streams are rejected + by Git. Locks cover each native mutation rather than the writer object's + lifetime; replacing a section is not one atomic transaction. * ``Submodule.rename()`` is removed. Moving a submodule preserves its logical name, matching Git. Normal removal retains Git's recoverable submodule metadata. Re-adding with ``no_checkout=True`` can reuse that metadata without changing refs; incompatible URLs, branches, and clone-only options are rejected before mutation. Fetch/pull results are derived from command output rather than ``FETCH_HEAD`` - file parsing. Revision strings follow Git's native revision grammar. + file parsing. Revision strings follow Git's native revision grammar. Root trees + such as ``Repo.rev_parse("HEAD:")`` have an empty path and can be iterated. + Missing paths within a valid tree revision raise ``Tree.join()``'s ``KeyError`` + with its missing-component message. * ``IndexFile.entries`` returns an immutable snapshot keyed by ``(path, stage)`` for compatibility with existing readers. Each access queries the current @@ -166,8 +159,8 @@ API changes * Commit/tag metadata uses structured Gix decoders when the Gix backend supports the repository. The CLI backend retains raw decoding because Git has no faithful formatted query for all commit headers or arbitrary tag objects. ``gpgsig`` - remains readable. The small ``Actor.from_string()`` and - ``parse_actor_and_date()`` identity helpers remain supported. + remains readable. ``Actor.from_string()`` and ``parse_actor_and_date()`` provide + identity parsing. * ``parse_date()`` delegates text syntax and timezone interpretation to Git; aware datetimes retain direct conversion and invalid inputs raise ``ValueError``. ISO/RFC dates now apply their timezone to the UTC timestamp, correcting the @@ -182,9 +175,10 @@ API changes ``Actor.from_string``), ``git.util.Iterable`` (use ``IterableObj``), ``git.compat.is_win/is_posix/is_darwin`` (use ``os.name``/``sys.platform``), and ``git.types.Lit_commit_ish`` (use ``Literal["commit", "tag"]`` or - ``GitObjectTypeString``). Top-level typing exports and private module aliases - are removed; import from ``typing`` or the owning module. ``git.util`` now - exposes the actual utility module. Abstract ``Traversable.traverse`` and + ``GitObjectTypeString``). Top-level typing exports other than explicit + ``Union`` and ``Optional`` imports are removed, along with private module + aliases; import from ``typing`` or the owning module. ``git.util`` now exposes + the actual utility module. Abstract ``Traversable.traverse`` and ``list_traverse`` raise ``NotImplementedError``; use concrete implementations. * Paths and URLs no longer expand ``$VAR``/``%VAR%`` automatically. ``Repo`` still accepts ``expand_vars=False`` for callers that explicitly select literal @@ -1145,10 +1139,6 @@ Please note that due to breaking changes, we have to increase the major version. * CRITICAL: fixed incorrect `Commit` object serialization when authored or commit date had timezones which were not divisiblej by 3600 seconds. This would happen if the timezone was something like `+0530` for instance. * A list of all additional fixes can be found `on GitHub `__ -* Historical correction: ``Tree.cache`` was announced as removed, but the interface - remained available. Its current behavior and limitations are documented in - :class:`~git.objects.tree.TreeModifier`. Prefer ``IndexFile.write_tree()`` when - building trees from the staging area. 1.0.1 - Fixes ============= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 9bdc8b0fc..1faa3f922 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -6,7 +6,7 @@ Overview / Install GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. -It provides Python objects for repository data and delegates Git operations to the Git executable. The default backend supports both SHA-1 and SHA-256 object IDs and both files and reftable reference storage. The legacy pure-Python GitDB backend is removed. +It provides Python objects for repository data and delegates Git operations to the Git executable. The default backend supports both SHA-1 and SHA-256 object IDs and both files and reftable reference storage. ``GitDB`` is an alias for ``GitCmdObjectDB``; the pure-Python storage implementation is unavailable. The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index a56d2ac0c..fdcb156a4 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -548,9 +548,10 @@ resolves abbreviated object IDs through persistent ``git cat-file`` processes:: # Equivalent explicit selection: repo = Repo("path/to/repo", odbt=GitCmdObjectDB) -Only ``GitCmdObjectDB`` and subclasses can be selected through ``odbt``. The -legacy ``GitDB`` backend is removed. The ``gitdb`` package remains a dependency -for shared types and utilities. +Only ``GitCmdObjectDB`` and subclasses can be selected through ``odbt``. +``GitDB`` is an alias for ``GitCmdObjectDB``; passing ``odbt=GitDB`` selects the +same backend. The pure-Python storage implementation is unavailable. The +``gitdb`` package provides shared types and utilities. Git Command Debugging and Customization *************************************** diff --git a/fuzzing/README.md b/fuzzing/README.md index 0d557d658..925740c9a 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -71,8 +71,8 @@ Contains Python files for each fuzz test. **Things to Know**: - Each fuzz test targets a specific part of GitPython's functionality. -- Repository and object targets exercise Git-backed workflows. Removed Python index/tree binary parsers are no longer - fuzz targets; malformed configuration and object IDs can now be rejected by Git or by Python input validation. +- Repository and object targets exercise Git-backed workflows. Git or Python input validation may reject malformed + configuration and object IDs. - Test files adhere to the naming convention: `fuzz_.py`, where `` indicates the functionality targeted by the test. - Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but diff --git a/git/db.py b/git/db.py index bc812a749..63dcd9f5f 100644 --- a/git/db.py +++ b/git/db.py @@ -109,6 +109,5 @@ def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: raise BadObject(partial_hexsha) from exc -# Preserve the public object-backend selector used by downstream applications. -# Storage remains owned by Git/Gix; this does not restore gitdb's Python backend. +# Keep the public backend selector using the same Git/Gix-managed storage. GitDB = GitCmdObjectDB diff --git a/git/objects/tree.py b/git/objects/tree.py index f9361558e..f07cad28a 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -57,10 +57,9 @@ def cmp(a: str, b: str) -> int: class TreeModifier: """Edit the entries cached by one :class:`Tree` instance. - This interface is supported; it is not deprecated. Changes affect only that - instance's in-memory entries. They do not update its object ID, the stored tree, - parent trees, the index, or any commit. Call :meth:`set_done` to restore entry - order for iteration after editing. + Changes affect only that instance's in-memory entries. They do not update its + object ID, the stored tree, parent trees, the index, or any commit. Call + :meth:`set_done` to restore entry order for iteration after editing. Tree serialization delegates construction and ordering to Git or the supported Gix backend. Prefer :class:`~git.index.base.IndexFile` and diff --git a/git/repo/base.py b/git/repo/base.py index f03a52b44..0aace28f3 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -266,7 +266,8 @@ def __init__( Object DataBase type - a type which is constructed by providing the directory containing the database objects, i.e. ``.git/objects``. It will be used to access all object data. Only :class:`~git.db.GitCmdObjectDB` - and its subclasses are supported. + and its subclasses are supported. ``GitDB`` is an alias for + ``GitCmdObjectDB``. :param search_parent_directories: If ``True``, all parent directories will be searched for a valid repo as @@ -276,9 +277,8 @@ def __init__( GitPython, which is considered a bug though. :param expand_vars: - The legacy ``False`` option remains accepted to select literal paths. - ``True`` is unsupported; expand environment variables explicitly - before opening a repository. Initial ``~`` expansion remains enabled. + Only ``False`` is supported. Expand environment variables explicitly + before opening a repository. Initial ``~`` expansion is enabled. :raise git.exc.InvalidGitRepositoryError: @@ -1602,7 +1602,8 @@ def init( Object DataBase type - a type which is constructed by providing the directory containing the database objects, i.e. ``.git/objects``. It will be used to access all object data. Only :class:`~git.db.GitCmdObjectDB` - and its subclasses are supported. + and its subclasses are supported. ``GitDB`` is an alias for + ``GitCmdObjectDB``. :param allow_unsafe_options: Allow unsafe options to be used, such as ``--template`` and @@ -1786,8 +1787,8 @@ def clone( :param kwargs: * ``odbt`` = ObjectDatabase Type, allowing to determine the object database implementation used by the returned :class:`Repo` instance. The - backend must be ``GitCmdObjectDB`` or its subclass; use the default - :class:`~git.db.GitCmdObjectDB` instead. + backend must be :class:`~git.db.GitCmdObjectDB` or its subclass. + ``GitDB`` is an alias for ``GitCmdObjectDB``. * All remaining keyword arguments are given to the :manpage:`git-clone(1)` command. diff --git a/test/downstream/audit.md b/test/downstream/audit.md index 340e8d05d..e58dc2e73 100644 --- a/test/downstream/audit.md +++ b/test/downstream/audit.md @@ -2,14 +2,14 @@ ## Known incompatibilities -The October 22–23, 2026 audit covered 115 projects on both CLI and Gix backends. -These differences remain after the audit fixes; later passing selections do not close them. -Released GitPython **3.2.1** is the API compatibility baseline. +The October 22–23, 2026 audit covered 115 projects on both CLI and Gix backends +of the unreleased migration. These incompatibilities are relative to released +GitPython **3.2.1**; passing selections do not close them. ### Read-only index entries `IndexFile.entries` is an immutable `(path, stage)` snapshot. Readers used by dlt -and Renku work again, but dictionary mutation remains unsupported. Callers that +and Renku can read these snapshots; dictionary mutation is unsupported. Callers that write entries directly must use supported index operations. This is an intentional API break, not a demonstrated Gitoxide bug; the audit does not count an untested writer as passing. @@ -29,7 +29,7 @@ It calls `add_section('gpg "ssh"')` before assigning `allowedSignersFile` throug The available Git/Gix config edit APIs do not create empty sections, and the inherited ConfigParser section API was removed. Callers can use `set_value('gpg "ssh"', 'allowedSignersFile', path)` directly. Nonempty section -assignment is restored; empty-section support remains a missing capability/API break. +assignment is supported; empty-section support is a missing capability/API break. ### Mistral Vibe: repository finalization timing @@ -73,30 +73,32 @@ incorrect. Four other real LFS/helper supplements pass. Its five branch incompat are separate from these ten outcomes. Initial dependency, fixture and quota errors were kept separately from corrected final runs; source expectations were unchanged. -## Compatibility fixes established by the audit +## Verified compatibility -Regressions failed before the corresponding fixes. Counts below remain the recorded -audit results, rather than a new run of these downstreams against the current branch head. +The commits identify unreleased development revisions used to verify these contracts. +Counts are the recorded audit results, rather than a new run of these downstreams +against the current branch head. -| Commit | Affected caller / correction | +| Commit | Affected caller / verified behavior | | --- | --- | -| `b3a4ba8d` | dlt/Renku: restore immutable `IndexFile.entries` readers, including merge stages; writers remain unsupported. | -| `2fd0d008` | Mistral Vibe: stop automatic `GIT_DIR`/`GIT_WORK_TREE` from overriding global `-C`; preserve explicit environment precedence. Fixes 22 cases and avoids staging in the original dirty checkout. | -| `8ce71f5c` | Mistral Vibe: preserve checkout context for separate Git directories; fixes two staging cases using Git/Gix-resolved worktree metadata. | -| `97921e3e` | Aider: restore `git.GitDB`/`git.db.GitDB` as selectors for managed `GitCmdObjectDB`; all 21 repository tests pass. | +| `b3a4ba8d` | dlt/Renku: support immutable `IndexFile.entries` readers, including merge stages; writers are unsupported. | +| `2fd0d008` | Mistral Vibe: respect global `-C` despite automatic `GIT_DIR`/`GIT_WORK_TREE`; preserve explicit environment precedence. The 22 affected cases pass, with staging isolated from the original dirty checkout. | +| `8ce71f5c` | Mistral Vibe: preserve checkout context for separate Git directories; two staging cases pass using Git/Gix-resolved worktree metadata. | +| `97921e3e` | Aider: accept `git.GitDB`/`git.db.GitDB` as selectors for managed `GitCmdObjectDB`; all 21 repository tests pass. | | `8877d095` | Angr: resolve branches tracking another local branch through Git's upstream resolver; five real checks pass. Released GitPython also had this bug; the operation retains a CLI fallback. | -| `44403026` | MCP Server Git: restore root-tree paths and missing-component `KeyError` diagnostics; three regressions fixed, complete 186-test selection passes. | -| `47a668ce` | sbxloop: accept explicit `expand_vars=False` in its original constructor position; fixes 91 cases. Paths remain literal and `True` rejects automatic expansion. | +| `44403026` | MCP Server Git: preserve root-tree paths and missing-component `KeyError` diagnostics; three focused regressions and the complete 186-test selection pass. | +| `47a668ce` | sbxloop: accept explicit `expand_vars=False` in its original constructor position; 91 affected cases pass. Paths are literal and `True` rejects automatic expansion. | | `2d84a6ae` | Salt: resolve the editable/source `git.__version__` from `VERSION`; provider detection and 34 unchanged tests pass. | | `a88e46a4` | Packaging: keep that source-only version fallback inactive in wheels; isolated installation profiles pass. | | `617a0586` | Demisto SDK: retain the explicit `typing.Union` import from `git`; validator registration and 25 upstream tests pass. | -| `5749f6db` | Kas: restore nonempty config section assignment through Git config remove/set operations; 11 real supplements pass. Values are validated first; replacement is not transaction-wide atomic. | +| `5749f6db` | Kas: support nonempty config section assignment through Git config remove/set operations; 11 real supplements pass. Values are validated first; replacement is not transaction-wide atomic. | | `3640db47` | MkDocs RSS Plugin: retain the explicit `typing.Optional` import from `git`; five upstream and ten supplemental cases pass. | Scripted regressions in [test_downstream_compatibility.py](../test_downstream_compatibility.py) cover SHA-1/SHA-256, files/reftable, index snapshots, revision paths, linked worktrees, explicit environments, literal paths, and separate Git directories. Config and -installation regressions cover the later fixes. Git/Gix retain ownership and input validation. +installation regressions cover section assignment, typing imports and source versions. +Git/Gix retain ownership and input validation. ## Selection, method and recorded totals @@ -217,7 +219,7 @@ those suites nor the 115 downstream audits; later branch changes have their own | siliconcompiler | 0.38.11 | 264 | 8 | 0 | 0 | Resolver/updater/issue suite includes clones/metadata plus many mocks; real clone/checkout/cache/submodule/version supplements; EDA tools omitted. | | pyats-reporter | 26.9 | 0 | 9 | 0 | 0 | Actual closed-source Cython get_git_info helper on real repositories; verified aarch64 wheel; no public source or upstream suite. | | zenml | 0.98.0 | 3 | 16 | 0 | 0 | 3 upstream CodeArchive mocks; 16 real clone/repository-context/archive checks; onboarding/Jupyter and ML pipelines omitted. | -| mcp-server-git | 2026.10.19 | 186 | 0 | 0 | 0 | Complete unchanged server/protocol/eras/entrypoint suite; real Git tools and protocol calls; 3 revision-path regressions fixed. | +| mcp-server-git | 2026.10.19 | 186 | 0 | 0 | 0 | Complete unchanged server/protocol/eras/entrypoint suite; real Git tools and protocol calls; 3 revision-path regressions pass. | | cruft | 2.16.0 | 182 | 0 | 0 | 0 | Complete API/CLI/utilities selection with real template clones, recursive submodules and updates; subprocess display abbreviation normalized. | | sbxloop | 2.1.36 | 210 | 4 | 15 | 0 | 224 upstream cases: 210 pass, 5 remaining compatibility failures, 1 inherited assertion, 8 LFS PATH setup failures; 4 supplemental pass and 1 bounded LFS timeout. | @@ -227,7 +229,7 @@ those suites nor the 115 downstream audits; later branch changes have their own | --- | --- | ---: | ---: | ---: | ---: | --- | | duplocloud-client | 0.4.5 | 0 | 12 | 0 | 0 | 12 real build/version helper checks; no upstream Git tests; whole unchanged module. | | phidata | 2.7.10 | 0 | 17 | 0 | 0 | 17 real origin/workspace clone/progress checks; no upstream Git tests; final cloud setup boundary replaced. | -| salt | 3008.3 | 34 | 0 | 0 | 0 | 34 unchanged real GitFS/fileserver/pillar cases; provider detection fixed; 50 alternative-provider cases deselected. | +| salt | 3008.3 | 34 | 0 | 0 | 0 | 34 unchanged real GitFS/fileserver/pillar cases; provider detection accepts the source-checkout version; 50 alternative-provider cases deselected. | | tinybird | 5.0.2 | 0 | 0 | 0 | 0 | Unused declaration: verified source and wheel have no GitPython caller; current Git paths use subprocess. | | demisto-sdk | 1.39.13 | 25 | 20 | 1 | 0 | 25 upstream cases, five real and 20 mocked/helper; 20 real supplemental passes; root listing also fails with the release. | | langchain-cli | 0.0.37 | 4 | 8 | 0 | 0 | Four upstream parsing cases and eight real clone/pull/cache supplements of the whole helper module. | @@ -252,13 +254,13 @@ those suites nor the 115 downstream audits; later branch changes have their own | opentimestamps-client | 0.7.2 | 1 | 11 | 0 | 0 | One upstream hash-only test and 11 real offline extraction/tree/blob supplements; synthetic signature/pending proof; no calendar/GnuPG validation. | | robot-descriptions | 3.3.0 | 11 | 11 | 0 | 0 | 11 byte-identical upstream clone/progress cases and 11 real cache/tag/SHA/shallow supplements; local fixtures avoid assets; test initializer isolated. | | wily | 1.25.0 | 117 | 0 | 0 | 0 | 117 unchanged integration/archiver tests, two with Repo mocks; all pass with the release; branch/quota setup failures retained. | -| kas | 5.5 | 0 | 11 | 3 | 0 | 11 real diff/shallow/identity supplements pass after the fix; three upstream SSH empty-section compatibility failures retained. | +| kas | 5.5 | 0 | 11 | 3 | 0 | 11 real diff/shallow/identity supplements pass; three upstream SSH empty-section compatibility failures retained. | | exasol-integration-test-docker-environment | 6.5.1 | 0 | 7 | 0 | 0 | Seven real get_commit_id supplements; normal module without Docker constructor; no upstream Git tests or Docker workflow. | | comfy-cli | 1.23.0 | 20 | 19 | 0 | 0 | 20 upstream cases and 19 real repository/workspace installation checks; other Git helpers use subprocesses; no server/model actions. | | airbyte-internal-ops | 0.126.5 | 0 | 20 | 2 | 0 | 20 real supplements pass; two nested third-party path failures also released; whole utilities module and three AST helpers; cloud stack omitted. | | nipype | 1.12.0 | 0 | 7 | 0 | 0 | Seven real full Zenodo maintainer-script supplements; no upstream Git caller tests; neuroimaging stack omitted. | | sam3 | 0.1.4 | 0 | 0 | 0 | 0 | Unused development pin; 140 Python files have no direct/dynamic Git imports; zero tests; model/Torch stack omitted. | -| mkdocs-rss-plugin | 1.19.0 | 5 | 10 | 0 | 0 | Five unchanged date/no-repo builds and ten real history/meta/shallow/CiHandler supplements; Optional import fixed; normal modules. | +| mkdocs-rss-plugin | 1.19.0 | 5 | 10 | 0 | 0 | Five unchanged date/no-repo builds and ten real history/meta/shallow/CiHandler supplements; Optional import supported; normal modules. | | pytextrank | 3.3.0 | 0 | 16 | 0 | 0 | 16 real version/commit/tag supplements; complete version module imported by file; spaCy omitted; no upstream Git tests. | | apm-cli | 0.34.0 | 184 | 10 | 0 | 0 | 184 unchanged metadata/cache/contract cases and ten real clone/progress/detached/worktree/locked-HEAD supplements; no MCP/API services. | | hf-doc-builder | 0.5.0 | 0 | 16 | 0 | 0 | 16 real ls_remote/tag and local HTTP inventory supplements; normal external module; CLI-only Git caller; no upstream Git tests. | From 64e169c3d547a1d1045684d1a1d25178efdeed5d Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:06:02 +0100 Subject: [PATCH 12/21] refactor(gix): open fresh repository state for each operation Retained native repositories required file and environment fingerprints, conditional reloads and a per-Repo refresh lock, while included configuration still needed unconditional refresh. Open a new `gix.Repository` through the shared accessor for each operation and remove `_gix_state`, the cached handle, the refresh lock and CLI-triggered invalidation. Keep execution-only configuration overrides out of configuration queries and preserve storage compatibility fallbacks. Update pickling, cleanup assertions and documentation for operation-local native handles. Reopening repeats native initialization; the historical retained-handle timings do not measure this policy. The ten focused Gix lifetime/discovery cases pass. Cleanup, fixture lifetime and the documented close-before-removal example pass on both backends (27 tests and one skip each), with localhost access for the Git daemon cases. Ruff lint and format checks pass. Benchmark timings were not rerun. --- README.md | 8 ++--- doc/gix-backend.md | 70 +++++++++++++++++-------------------- git/_backend.py | 43 ++--------------------- git/cmd.py | 5 +-- git/repo/base.py | 41 ++++++---------------- test/performance/README.md | 4 +-- test/test_cleanup.py | 2 +- test/test_docs.py | 1 - test/test_fixture_health.py | 1 - test/test_gix_backend.py | 44 ++++++++++------------- 10 files changed, 70 insertions(+), 149 deletions(-) diff --git a/README.md b/README.md index 6f740f7d9..3974060f7 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,9 @@ bindings to execute supported operations through Gitoxide while exposing GitPyth high-level repository, commit, tree and index interfaces. Supported object reads, tree traversal, reference/index queries and commit history -can run without launching Git subprocesses. GitPython also retains and reuses a Gix -repository handle across operations, reducing repeated initialization work. These -changes reduce subprocess overhead, particularly for repeated repository queries; -the [backend benchmarks](test/performance/README.md) measure results and CLI launch -counts for both backends. +can run without launching Git subprocesses. Each native operation opens a fresh +Gix repository handle. The [backend benchmarks](test/performance/README.md) measure +results and CLI launch counts for both backends. ### Activate the Gix backend diff --git a/doc/gix-backend.md b/doc/gix-backend.md index 74cd8556d..172db6858 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -453,9 +453,9 @@ environment metadata. operations and their complete journey on one already-open `git.Repo`, plus separate direct opening and discovery from `git/objects`. Fresh high-level wrappers preserve the cost of actual operations; imports, fixture preparation -and parity preflight are outside timing. Native repository refresh remains -inside operation timing, so these measurements include the cost of keeping -retained state current. +and parity preflight are outside timing. Native repository opening remains inside +operation timing. Historical retained-handle results below included refresh +costs instead. Submodule inventory, revision path/mode lookup and raw commit/tree readback extend the already-open journey, each with a zero-CLI ceiling. The readback @@ -632,8 +632,8 @@ used Gix; CLI validation used the affected selections. Repository-wide Ruff lint/format, mypy and basedpyright passed. The benchmark comparison independently checked result parity, fixture/source identity and all process ceilings. Generic configuration and standalone reference-name gaps remain documented -under GIX-12/20. `Repo._get_gix_repository(recreate=True)` still explicitly -recreates retained native state; its snapshot limitations are documented below. +under GIX-12/20. That snapshot retained native repository state; the current +accessor opens a fresh handle for each operation, as documented below. Raw benchmarks, comparison tables and logs from that snapshot are under `.cache/gix-only-audit/benchmark-*` and `.cache/gix-only-audit/pyperf-comparison.md`. @@ -785,39 +785,34 @@ expected failure. Operation reporting counted 114,800 native operations and 39,002 CLI fallback decisions. Those decisions are not a complete subprocess count: raw `repo.git` calls and subprocesses started by Git are not all counted. -### Retain native repository state (implemented) - -Each `git.Repo` owns a native `gix.Repository`. Managed operations access it through -`Repo._get_gix_repository()`, which reuses the instance by default. -`Repo._get_gix_repository(recreate=True)` replaces it explicitly; this method is -the control point for a future configurable reuse policy. Managed operations reuse it; -`close()` releases it, and a later operation can reopen it. Pickling excludes -native resources and restores the command's weak owner reference. Gix provides -thread-safe handle access and automatic index/ODB snapshot refresh. A per-Repo -lock serializes handle refresh, without serializing read operations. - -Retaining native state is an intentional deviation from Git's fresh process per -command. Configuration/environment and observed CLI changes trigger best-effort -refresh, but this is not a promise that every external filesystem change is -immediately visible in every native cache. Explicit recreation or closing and -reopening the `Repo` provides fresh native state. Configuration queries currently -use a separate fresh handle because the retained execution handle overrides -hooks, fsmonitor and automatic maintenance; those overrides must not leak into -configuration results. The accessor itself removes no CLI calls: ten supported -metadata queries launch zero processes before and after this refactor. - -Configuration and storage metadata changes, environment changes and raw CLI -launches cause `reload()` before reuse. Configuration queries use a separate -fresh handle so synthetic safety settings never appear as user settings. -Configurations with includes conservatively reload on each operation because -the binding does not expose all included source paths. Storage overrides, -reftable and compatibility object formats retain their existing CLI guards. +### Open fresh native repository state + +Managed operations access Gix through `Repo._get_gix_repository()`, which opens +a fresh `gix.Repository` for each call. Standalone commands use the same opening +path. No native handle, environment/file fingerprint or refresh lock is cached +on `Repo`; configuration, includes and storage metadata are read by Gix again +on each open. Concurrent operations therefore receive independent handles. + +Execution handles override hooks, fsmonitor and automatic maintenance. +Configuration queries omit those overrides so synthetic settings do not appear +as user settings. Storage overrides, reftable and compatibility object formats +retain their existing CLI guards. Reopening itself adds no Git CLI calls. + +`Repo.close()` releases persistent CLI object readers. Native handles remain +local to operations; callers holding separately opened handles or iterators +must release them before deleting their repository, especially on Windows. +Pickling restores the command's weak owner reference without native resources. + +This replaces the retained-handle policy and its best-effort invalidation via +file metadata, environment changes and CLI launches. Reopening repeats native +initialization work but avoids fingerprint construction and unconditional +reloads for configurations with includes. An earlier 1,000-call benchmark without coverage measured an object read using -one native handle at 0.046 ms per call, compared with 0.635 ms through the -fresh-open stream adapter. Opening and validating a native repository alone -took 0.418 ms per call. These small repeated-read measurements do not estimate -whole-suite speedup. +one retained native handle at 0.046 ms per call, compared with 0.635 ms through +the fresh-open stream adapter. Opening and validating a native repository alone +took 0.418 ms per call. Those historical measurements do not describe the current +accessor or estimate whole-suite speedup; rerun the backend harness to measure it. ### Reuse prepared test fixtures (implemented) @@ -1230,8 +1225,7 @@ can use Git to preserve its public diagnostics. Once `_write()` begins, `gix.Error` failures become `GitCommandError`; other exceptions propagate. Neither may trigger a second CLI mutation. Access native repository state through `Repo._get_gix_repository()` for bound -commands. Reuse is the default, with explicit recreation available; keep the -snapshot deviation and refresh behavior above documented. Close native iterators +commands. Each access opens a fresh handle through Gix. Close native iterators when partially consumed. Add each conversion with a no-subprocess check and a Git parity check where diff --git a/git/_backend.py b/git/_backend.py index 9d5fb2b45..6fbb3fe94 100644 --- a/git/_backend.py +++ b/git/_backend.py @@ -144,56 +144,17 @@ def _open_repository(command: Any, env: Dict[str, Any], *, query_config: bool = options = options.config_overrides( ["core.fsmonitor=false", "gc.auto=0", "maintenance.auto=false", "core.hooksPath=" + os.devnull] ) - owner = command._repo() if command._repo is not None else None - state = None - repo = None - if owner is not None and not query_config: - # Gix refreshes index/ODB snapshots itself; configuration is loaded at open. - paths = [ - path, - os.fspath(owner.common_dir), - os.path.join(owner.common_dir, "config"), - os.path.join(path, "config.worktree"), - ] - paths += [ - effective.get("GIT_CONFIG_SYSTEM", "/etc/gitconfig"), - effective.get("GIT_CONFIG_GLOBAL", os.path.expanduser("~/.gitconfig")), - os.path.join(effective.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "git", "config"), - os.path.join(owner.common_dir, "objects", "info", "alternates"), - os.path.join(path, "commondir"), - os.path.join(path, "gitdir"), - ] - stamps: List[Any] = [] - for filename in paths: - try: - info = os.stat(filename) - stamps.append((info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)) - except FileNotFoundError: - stamps.append(None) - state = (path, tuple(sorted(effective.items())), tuple(stamps)) - repo = owner._gix_repository - if repo is not None and os.path.realpath(repo.git_dir()) != os.path.realpath(path): - repo = None - if repo is not None and state != owner._gix_state: - repo.reload() - if repo is None: - repo = gix.open_opts(path, options) + repo = gix.open_opts(path, options) snapshot = repo.config_snapshot() if snapshot.string("extensions.refStorage") == b"reftable": raise _Unsupported("reftable (GIX-1)") if snapshot.string("extensions.compatObjectFormat") is not None: raise _Unsupported("compatibility object format (GIX-19)") - if effective.get("GIT_WORK_TREE") and (owner is None or query_config or state != owner._gix_state): + if effective.get("GIT_WORK_TREE"): workdir = effective["GIT_WORK_TREE"] if not os.path.isabs(workdir): workdir = os.path.join(command.working_dir or os.getcwd(), workdir) repo.set_workdir(workdir) - if owner is not None and not query_config: - owner._gix_repository = repo - # ponytail: includes can load arbitrary files; reopen until Gix exposes their source paths. - if state != owner._gix_state: - included = re.search(rb"\[include(?:if)?[\s\]]", repo.config_snapshot().plumbing().to_bstring(), re.I) - owner._gix_state = None if included else state return repo diff --git a/git/cmd.py b/git/cmd.py index 89788ae68..254c661c8 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1166,7 +1166,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None) -> None: # Extra environment variables to pass to git commands self._environment: Dict[str, Optional[str]] = {} self._repository_environment: Dict[str, Optional[str]] = {} - self._repo: Any = None # Weak reference; the Repo owns native resources. + self._repo: Any = None # Weak reference to the Repo's native repository accessor. # Cached version slots self._version_info = None @@ -1587,9 +1587,6 @@ def execute( raise GitCommandNotFound(redacted_command, err) from err else: _backend.record("Git.execute", "CLI process") - owner = self._repo() if self._repo is not None else None - if owner is not None: - owner._gix_state = None # Replace with a typeguard for Popen[bytes]? proc.stdout = cast(BinaryIO, proc.stdout) proc.stderr = cast(BinaryIO, proc.stderr) diff --git a/git/repo/base.py b/git/repo/base.py index 0aace28f3..0de87d01c 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -16,7 +16,6 @@ import shlex import sys import tempfile -from threading import RLock import weakref import gitdb @@ -125,8 +124,6 @@ class Repo: # Must exist, or __del__ will fail in case we raise on `__init__()`. git = cast("Git", None) - _gix_repository: Any = None - _gix_state: Any = None working_dir: PathLike """The working directory of the git command.""" @@ -460,7 +457,6 @@ def __init__( break self.working_dir = self._working_tree_dir or self.common_dir - self._gix_lock = RLock() self.git = self.GitCommandWrapperType(self.working_dir) self.git._repo = weakref.ref(self) self.git._environment.update(GIT_DIR=git_dir, **environment) @@ -471,7 +467,6 @@ def __init__( if "GIT_WORK_TREE" not in environment: self.git._repository_environment["GIT_WORK_TREE"] = os.fspath(self._working_tree_dir) if native is not NotImplemented: - self._gix_repository = native self._empty_tree_hexsha = str(native.empty_tree().id) else: with tempfile.TemporaryFile() as empty: @@ -494,32 +489,18 @@ def _get_gix_repository( command: Optional[Git] = None, env: Optional[Dict[str, Any]] = None, query_config: bool = False, - recreate: bool = False, ) -> Any: - """Access native state, reusing the repository unless recreation is requested. + """Open a fresh native repository for each operation. - This method owns the reuse policy so it can later become configurable. - Configuration queries currently open a separate handle without the execution - restrictions applied to the retained handle. Other operations retain the - existing best-effort refresh of configuration, environment and CLI changes. + Configuration queries omit execution restrictions so synthetic settings + do not appear in configuration results. """ - with self._gix_lock: - if recreate: - self._gix_repository = self._gix_state = None - return _backend._open_repository( - command if command is not None else self.git, env or {}, query_config=query_config - ) - - def __getstate__(self) -> Dict[str, Any]: - return { - key: value - for key, value in self.__dict__.items() - if key not in ("_gix_repository", "_gix_state", "_gix_lock") - } + return _backend._open_repository( + command if command is not None else self.git, env or {}, query_config=query_config + ) def __setstate__(self, state: Dict[str, Any]) -> None: self.__dict__.update(state) - self._gix_lock = RLock() self.git._repo = weakref.ref(self) def __enter__(self) -> "Repo": @@ -535,14 +516,14 @@ def __del__(self) -> None: pass def close(self) -> None: - """Release native repository state and persistent Git object readers. + """Release persistent Git object readers. Use ``with Repo(path) as repo:`` to close automatically, including on errors. - Close every repository instance and any separately opened readers before - removing its directory, especially on Windows where open handles prevent - deletion. :func:`git.util.rmtree` also handles Git's read-only files. + Close every repository instance and release any separately opened native + handles or readers before removing its directory, especially on Windows + where open handles prevent deletion. :func:`git.util.rmtree` also handles + Git's read-only files. """ - self._gix_repository = self._gix_state = None if self.git: self.git.clear_cache() # Tempfiles objects on Windows are holding references to open files until diff --git a/test/performance/README.md b/test/performance/README.md index eb53e267f..2fb13fd9c 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -7,8 +7,8 @@ instance open per worker, but creates fresh commit/tree/index wrappers per invocation so their cached properties cannot turn the measurement into a no-op. The CLI's persistent `cat-file` processes and the filesystem cache are warmed. Repository creation, imports, preflight and result hashing are outside timing. -Native handle refresh and compatibility checks remain inside timing, exposing -the benefit of retaining a Gix repository across operations. Opening and +Native repository opening and compatibility checks remain inside operation +timing; each native operation opens a fresh Gix handle. Opening and discovery retain explicit CLI fallbacks tracked in `cli-budget.json`. `open_repository` and `discover_repository` instead create and close a new `Repo` in each timed invocation; they do not join the already-open journey. diff --git a/test/test_cleanup.py b/test/test_cleanup.py index 3443c88ae..8cd20a13b 100644 --- a/test/test_cleanup.py +++ b/test/test_cleanup.py @@ -229,7 +229,7 @@ def case(self, *resources): assert (directory / "locked").exists() assert not (directory / "removable").exists() for repo in repositories: - assert repo._gix_repository is None + assert repo.git.cat_file_header is repo.git.cat_file_all is None def test_directory_decorator_preserves_failed_test_artifacts(tmp_path, monkeypatch): diff --git a/test/test_docs.py b/test/test_docs.py index 97166e1ea..ecbf525d5 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -24,7 +24,6 @@ def test_repo_context_releases_resources_before_removal(): assert repo.head.commit.tree["file"].data_stream.read() == b"contents\n" processes = [reader.proc for reader in (repo.git.cat_file_header, repo.git.cat_file_all) if reader] - assert repo._gix_repository is None assert repo.git.cat_file_header is repo.git.cat_file_all is None assert all(process.poll() is not None for process in processes) # [remove-repo-directory] diff --git a/test/test_fixture_health.py b/test/test_fixture_health.py index 61c159c48..63a1c6b8d 100644 --- a/test/test_fixture_health.py +++ b/test/test_fixture_health.py @@ -32,7 +32,6 @@ def test_scripted_writable_fixture_copies_and_closes_repository(): (root / "file").write_text("private") repo.index.add(["file"]) repo.index.commit("Private change", skip_hooks=True) - assert repo._gix_repository is None assert not root.exists() assert (baseline / "repo/file").read_text() == "initial" diff --git a/test/test_gix_backend.py b/test/test_gix_backend.py index cb3c72b9a..80c15c294 100644 --- a/test/test_gix_backend.py +++ b/test/test_gix_backend.py @@ -94,50 +94,44 @@ def test_native_worktree_inventory_includes_bare_main(repo): def test_native_repository_lifetime_and_refresh(repo): handle = repo._get_gix_repository() assert isinstance(handle, gix.Repository) - with patch.object(gix, "open_opts", side_effect=AssertionError("unexpected reopen")): - assert repo.git._call_process_safe("rev_parse", "--show-object-format") == repo.object_format - assert repo._gix_repository is handle - with ThreadPoolExecutor(max_workers=2) as pool: - assert list( - pool.map(lambda _: repo.git._call_process_safe("rev_parse", "--show-object-format"), range(2)) - ) == [ - repo.object_format, - repo.object_format, - ] + with patch.object(gix, "open_opts", wraps=gix.open_opts) as opened: + with patch.object(Git, "execute", side_effect=AssertionError("unexpected CLI call")): + assert repo._get_gix_repository() is not handle + assert repo.git._call_process_safe("rev_parse", "--show-object-format") == repo.object_format + with ThreadPoolExecutor(max_workers=2) as pool: + handles = list(pool.map(lambda _: repo._get_gix_repository(), range(2))) + assert handles[0] is not handles[1] + assert all( + current is not handle and str(current.object_hash()) == repo.object_format for current in handles + ) + assert opened.call_count == 4 repo.index.add(["file"]) first = repo.index.commit("native", skip_hooks=True) assert repo.commit().hexsha == first.hexsha repo.git.commit("--allow-empty", "-m", "CLI", "--no-verify", "--no-gpg-sign") assert repo.commit().message == "CLI\n" - assert repo._gix_repository is handle assert {(e.path, e.stage) for e in repo.index.iter_entries()} == {("file", 0)} with repo.config_writer() as writer: writer.set_value("core", "abbrev", "9") + writer.set_value("gc", "auto", "7") assert _backend._repository(repo.git, {}).config_snapshot().integer("core.abbrev") == 9 - assert repo._gix_repository is handle + assert repo._get_gix_repository().config_snapshot().integer("gc.auto") == 0 + assert repo._get_gix_repository(query_config=True).config_snapshot().integer("gc.auto") == 7 + with repo.git.custom_environment(GIT_WORK_TREE=str(Path(repo.working_dir).parent)): + assert repo._get_gix_repository().workdir() == Path(repo.working_dir).parent + assert repo._get_gix_repository().workdir() == Path(repo.working_tree_dir) include = Path(repo.working_dir).parent / "included" repo.git.config("include.path", str(include)) assert _backend._repository(repo.git, {}).config_snapshot().string("test.value") == b"first" include.write_text("[test]\nvalue = second\n") assert _backend._repository(repo.git, {}).config_snapshot().string("test.value") == b"second" - count = _backend.statistics().get(("Git.execute", "CLI process"), 0) - with patch.object(gix, "open_opts", wraps=gix.open_opts) as opened: - replacement = repo._get_gix_repository(recreate=True) - assert replacement is not handle - assert repo._get_gix_repository() is replacement - opened.assert_called_once() - assert _backend.statistics().get(("Git.execute", "CLI process"), 0) == count - with pickle.loads(pickle.dumps(repo)) as restored: - assert restored._gix_repository is None assert restored.commit().hexsha == repo.commit().hexsha - assert restored._gix_repository is not None + assert restored.git._repo() is restored repo.close() - assert repo._gix_repository is None assert repo.commit().message == "CLI\n" - assert repo._gix_repository is not handle def test_opening_and_discovery_use_gix_with_explicit_fallbacks(repo): @@ -179,7 +173,6 @@ def test_opening_and_discovery_use_gix_with_explicit_fallbacks(repo): assert opened.object_format == repo.object_format assert opened.ref_format == "files" assert opened.bare is expected_bare - assert isinstance(opened._gix_repository, gix.Repository) assert opened._empty_tree_hexsha == repo._empty_tree_hexsha assert opened.git._call_process_safe("rev_parse", "--show-object-format") == repo.object_format assert ( @@ -236,7 +229,6 @@ def test_gitfile_reopens_canonical_native_repository(repo, target): with patch.object(Git, "execute", autospec=True, side_effect=Git.execute) as cli: with Repo(gitfile) as opened: assert (opened.git_dir, str(opened.working_tree_dir), opened.bare) == expected - assert isinstance(opened._gix_repository, gix.Repository) assert all("version" in call.args[1] or "--show-ref-format" in call.args[1] for call in cli.call_args_list) From b195d86774e56da42c2adf7730bcbdbeea1a8eac Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:13:27 +0100 Subject: [PATCH 13/21] fix(diff): preserve bytes in text paths and rename metadata Diff raw paths preserved bytes, but their text properties decoded with replacement characters. Renaming `legacy-\xff` to `legacy-\xfe` collapsed both names to the same string, hid the rename and gave diff blobs unusable paths. Patch rename fields also retained Git's quoted spelling. Use the existing lossless decoder for text paths and submodule matching, compare raw rename names, and unquote patch rename fields through the existing path decoder. Document the intentional change from replacement characters to surrogate escapes in public text paths. The new raw/patch rename regressions fail before this change and pass with both `core.quotePath` settings on CLI and Gix. Git 2.54.0 supplies the fixture renames; native operation counts confirm the raw Gix path. All 49 Diff tests pass on each backend. Ruff, ShellCheck and whitespace checks pass. --- doc/source/changes.rst | 4 ++++ git/diff.py | 18 +++++++++--------- test/test_diff.py | 19 ++++++++++++++++++- tests/fixtures/diff-byte-paths.sh | 11 +++++++++++ 4 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/diff-byte-paths.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 9b2aaf3f0..992e690ba 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -57,6 +57,10 @@ API changes * Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``, and raw rename fields. Staged-file diffs no longer fail while decoding legacy filenames; blob contents remain byte streams regardless of their encoding. + Text paths and rename properties use ``surrogateescape`` instead of replacing + undecodable bytes. Diff blob paths can be used for tree lookup, and renames + between distinct byte names are detected correctly. Patch rename fields + contain unquoted filename bytes. * Tree serialization now delegates construction to Git or the supported Gix backend. ``Tree.cache`` and ``TreeModifier`` edit one tree's in-memory entries; object IDs and parent trees do not change automatically. Prefer diff --git a/git/diff.py b/git/diff.py index a5c1a91b9..0118c3662 100644 --- a/git/diff.py +++ b/git/diff.py @@ -10,7 +10,7 @@ from git import _backend from git.cmd import Git, handle_process_output -from git.compat import defenc +from git.compat import defenc, safe_decode from git.objects.base import IndexObject from git.objects.blob import Blob from git.objects.util import mode_str_to_int @@ -486,7 +486,7 @@ def __init__( # we need to overwrite "repo" to the corresponding submodule's repo instead. if repo and a_rawpath and (self.a_mode == 0o160000 or self.b_mode == 0o160000): for submodule in repo.submodules: - if submodule.path == a_rawpath.decode(defenc, "replace"): + if submodule.path == safe_decode(a_rawpath): if submodule.module_exists(): repo = submodule.module() break @@ -582,24 +582,24 @@ def __str__(self) -> str: @property def a_path(self) -> Optional[str]: - return self.a_rawpath.decode(defenc, "replace") if self.a_rawpath else None + return safe_decode(self.a_rawpath) if self.a_rawpath else None @property def b_path(self) -> Optional[str]: - return self.b_rawpath.decode(defenc, "replace") if self.b_rawpath else None + return safe_decode(self.b_rawpath) if self.b_rawpath else None @property def rename_from(self) -> Optional[str]: - return self.raw_rename_from.decode(defenc, "replace") if self.raw_rename_from else None + return safe_decode(self.raw_rename_from) if self.raw_rename_from else None @property def rename_to(self) -> Optional[str]: - return self.raw_rename_to.decode(defenc, "replace") if self.raw_rename_to else None + return safe_decode(self.raw_rename_to) if self.raw_rename_to else None @property def renamed_file(self) -> bool: """:return: ``True`` if the blob of our diff has been renamed""" - return self.rename_from != self.rename_to + return self.raw_rename_from != self.raw_rename_to @classmethod def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_match: bytes) -> Optional[bytes]: @@ -742,8 +742,8 @@ def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> N new_file, deleted_file, copied_file, - rename_from, - rename_to, + decode_path(rename_from, has_ab_prefix=False) if rename_from else None, + decode_path(rename_to, has_ab_prefix=False) if rename_to else None, None, None, None, diff --git a/test/test_diff.py b/test/test_diff.py index 9426b78fc..5f3e44b80 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -9,6 +9,7 @@ import pytest from git import NULL_TREE, Diff, DiffIndex, Diffable, Repo +from git.compat import safe_encode from git.diff import decode_path from git.exc import UnsafeOptionError @@ -36,6 +37,22 @@ def test_raw_diff_preserves_non_utf8_paths(change): assert (diffs[0].raw_rename_from, diffs[0].raw_rename_to) == (before, after) +@pytest.mark.parametrize("create_patch", [False, True]) +@pytest.mark.parametrize("quote_path", ["false", "true"]) +def test_diff_paths_and_renames_preserve_non_utf8_bytes(create_patch, quote_path): + with fixture_repo("diff-byte-paths.sh", quote_path) as repo: + before, after = repo.commit("HEAD^"), repo.head.commit + (diff,) = before.diff(after, create_patch=create_patch) + assert (diff.a_rawpath, diff.b_rawpath) == (b"legacy-\xff", b"legacy-\xfe") + assert (diff.raw_rename_from, diff.raw_rename_to) == (diff.a_rawpath, diff.b_rawpath) + assert (safe_encode(diff.a_path), safe_encode(diff.b_path)) == (diff.a_rawpath, diff.b_rawpath) + assert (safe_encode(diff.rename_from), safe_encode(diff.rename_to)) == (diff.a_rawpath, diff.b_rawpath) + assert diff.renamed_file + if not create_patch: # Exact-rename patches omit the blob IDs. + assert before.tree[diff.a_blob.path].binsha == diff.a_blob.binsha + assert after.tree[diff.b_blob.path].binsha == diff.b_blob.binsha + + @ddt.ddt class TestDiff(TestBase): def setUp(self): @@ -351,7 +368,7 @@ def test_diff_unsafe_paths(self): self.assertEqual(res[8].b_path, "path/¯\\_(ツ)_|¯") self.assertEqual(res[9].b_path, "path/💩.txt") self.assertEqual(res[9].b_rawpath, b"path/\xf0\x9f\x92\xa9.txt") - self.assertEqual(res[10].b_path, "path/�-invalid-unicode-path.txt") + self.assertEqual(safe_encode(res[10].b_path), b"path/\x80-invalid-unicode-path.txt") self.assertEqual(res[10].b_rawpath, b"path/\x80-invalid-unicode-path.txt") # The "Moves" diff --git a/tests/fixtures/diff-byte-paths.sh b/tests/fixtures/diff-byte-paths.sh new file mode 100644 index 000000000..217b7b7db --- /dev/null +++ b/tests/fixtures/diff-byte-paths.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main repo +cd repo +git config core.quotePath "$1" +blob=$(printf 'content\n' | git hash-object -w --stdin) +before=$(printf '100644 blob %s\tlegacy-\377\0' "$blob" | git mktree -z) +after=$(printf '100644 blob %s\tlegacy-\376\0' "$blob" | git mktree -z) +parent=$(printf 'before\n' | git commit-tree "$before") +commit=$(printf 'after\n' | git commit-tree "$after" -p "$parent") +git update-ref refs/heads/main "$commit" From 037710e4ae41c2baae592cccfea383db0cdcf2a1 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:13:51 +0100 Subject: [PATCH 14/21] fix(index): preserve undecodable symlink target bytes Staging a symbolic link decoded its target through `os.readlink()` and then strictly re-encoded it. Targets containing undecodable filesystem bytes raised `UnicodeEncodeError` instead of being stored as the link blob. Use `os.fsencode()` to recover the original target bytes. Git's `object-file.c` stores the bytes read from a symlink directly; the scripted regression stages `../target-\xff` and verifies those exact object bytes. The fixture executes at its final location because native fixture copying follows symlinks. The regression fails before the fix and passes on both backends. Connected index tests pass on CLI and Gix; the combined Gix Diff/tree/index run passes 253 tests and nine subtests, with two platform skips. Ruff and ShellCheck pass. --- git/index/base.py | 5 ++--- test/test_index.py | 11 +++++++++++ tests/fixtures/index-byte-symlink.sh | 5 +++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/index-byte-symlink.sh diff --git a/git/index/base.py b/git/index/base.py index 28f685c94..61316df45 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -23,7 +23,7 @@ from gitdb.base import IStream from git import _backend -from git.compat import defenc, force_bytes, safe_decode +from git.compat import defenc, safe_decode from git.cmd import Git import git.diff as git_diff from git.exc import CheckoutError, GitCommandError, GitError, InvalidGitRepositoryError, UnmergedEntriesError @@ -754,8 +754,7 @@ def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry stream_size = st.st_size if S_ISLNK(st.st_mode): - # readlink is a string, but we need bytes. - target = force_bytes(os.readlink(filepath), encoding=defenc) + target = os.fsencode(os.readlink(filepath)) stream_size = len(target) def open_stream() -> BinaryIO: diff --git a/test/test_index.py b/test/test_index.py index d4db16ffb..f1f875443 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -52,6 +52,17 @@ def test_staged_legacy_filename_and_blob_bytes(): } +@pytest.mark.skipif(os.name == "nt", reason="Windows symlink targets are Unicode paths") +def test_staging_preserves_non_utf8_symlink_targets(): + if not symlinks_supported(): + pytest.skip("Symlinks unavailable") + # Native fixture copying follows symlinks instead of preserving their targets. + with fixture_repo("index-byte-symlink.sh", write=True, execute=True) as repo: + (entry,) = repo.index.add(["link"]) + assert entry.mode == 0o120000 + assert repo.odb.stream(entry.binsha).read() == b"../target-\xff" + + @ddt.ddt class TestIndex(TestBase): @with_rw_repo("HEAD") diff --git a/tests/fixtures/index-byte-symlink.sh b/tests/fixtures/index-byte-symlink.sh new file mode 100644 index 000000000..8b1bd26b3 --- /dev/null +++ b/tests/fixtures/index-byte-symlink.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -eu +gix_testtools_require_symlinks +git init --initial-branch=main repo +ln -s "$(printf '../target-\377')" repo/link From 45df03af1fda5891322415b674725e67fa2838b8 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:13:55 +0100 Subject: [PATCH 15/21] fix(tree): order cached entries by filename bytes `TreeModifier.set_done()` sorted decoded Unicode names. A name ending in an undecodable byte could sort before a valid UTF-8 name even when Git stores them in the opposite order, changing iteration order after cache editing. Recover each name's original bytes in the existing sort key and keep its directory terminator. This matches the byte comparison in Git's `tree.c::base_name_compare()` without changing tree construction, which still belongs to Git or Gix. The regression compares cache iteration with `git ls-tree -z` for byte and emoji names. It fails before the fix and passes on both backends. Connected tree tests pass on CLI and Gix; Ruff, ShellCheck and whitespace checks pass. --- doc/source/changes.rst | 3 ++- git/objects/tree.py | 4 ++-- test/test_tree.py | 9 +++++++++ tests/fixtures/tree-byte-order.sh | 12 ++++++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/tree-byte-order.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 992e690ba..d2c68d735 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -63,7 +63,8 @@ API changes contain unquoted filename bytes. * Tree serialization now delegates construction to Git or the supported Gix backend. ``Tree.cache`` and ``TreeModifier`` edit one tree's in-memory entries; - object IDs and parent trees do not change automatically. Prefer + ``set_done()`` orders entries by filename bytes, including undecodable names. + Object IDs and parent trees do not change automatically. Prefer ``IndexFile.write_tree()`` for changes to staged paths. * ``Commit.stats`` preserves Unicode line separators in filenames when ``core.quotePath=false``. Git's LF-delimited output remains intact for root, diff --git a/git/objects/tree.py b/git/objects/tree.py index f07cad28a..bbb48320b 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -10,7 +10,7 @@ import git.diff as git_diff from git.util import IterableList, join_path, to_bin_sha -from git.compat import defenc, safe_decode +from git.compat import defenc, safe_decode, safe_encode from git.cmd import Git from . import util @@ -91,7 +91,7 @@ def set_done(self) -> "TreeModifier": :return: self """ - self._cache.sort(key=lambda x: (x[2] + "/") if x[1] == Tree.tree_id << 12 else x[2]) + self._cache.sort(key=lambda x: safe_encode(x[2]) + (b"/" if x[1] == Tree.tree_id << 12 else b"")) return self # } END interface diff --git a/test/test_tree.py b/test/test_tree.py index 3cc2de776..f5273aa26 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -12,6 +12,7 @@ from gitdb import IStream +from git.compat import safe_encode from git.objects import Blob, Tree from git.objects.tree import TreeModifier from test.lib import TestBase, with_rw_repo @@ -55,6 +56,14 @@ def test_invalid_tree_does_not_retain_partial_cache(object_format, invalid_posit list(tree) +def test_tree_modifier_orders_non_utf8_names_as_git_does(): + with fixture_repo("tree-byte-order.sh") as repo: + expected = (read_only_fixture("tree-byte-order.sh") / "order").read_bytes().split(b"\0")[:-1] + tree = repo.head.commit.tree + tree.cache.set_done() + assert [safe_encode(item.name) for item in tree] == expected + + @ddt.ddt class TestTree(TestBase): @with_rw_repo("HEAD") diff --git a/tests/fixtures/tree-byte-order.sh b/tests/fixtures/tree-byte-order.sh new file mode 100644 index 000000000..decb84c84 --- /dev/null +++ b/tests/fixtures/tree-byte-order.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main repo +cd repo +blob=$(printf 'content\n' | git hash-object -w --stdin) +tree=$({ + printf '100644 blob %s\tname-\377\0' "$blob" + printf '100644 blob %s\tname-\360\237\222\251\0' "$blob" +} | git mktree -z) +commit=$(printf 'initial\n' | git commit-tree "$tree") +git update-ref refs/heads/main "$commit" +git ls-tree --name-only -z HEAD >../order From 072a6b4e3660d635c67b68513288b31686a7fee2 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:14:39 +0100 Subject: [PATCH 16/21] fix(config): preserve bytes in subsection names and values Git accepts configuration subsection names and values containing bytes that are not valid in the filesystem encoding. Strict decoding made configuration enumeration, remote names and URLs fail with `UnicodeDecodeError`; byte-valued writes also decoded strictly before reaching Git. Use the shared lossless decoder for Git's NUL-framed records and incoming byte values. Keep existing key validation, mutation framing and Git-managed locking. File and BytesIO regressions compare against `git config --null`, exercise remote names/URLs and verify exact bytes after writes. Four regressions fail before the fix. All 66 configuration tests pass on each backend with Git 2.54.0 and GixPython 0.3.0. Ruff, mypy and whitespace checks pass. Windows byte-valued argument writes remain excluded because Windows command-line arguments cannot preserve arbitrary original bytes. --- doc/source/changes.rst | 1 + git/config.py | 8 ++++---- test/test_config.py | 36 ++++++++++++++++++++++++++++++++++ tests/fixtures/config-bytes.sh | 5 +++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/config-bytes.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d2c68d735..dbccd79ea 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -125,6 +125,7 @@ API changes including updates when ``logmsg`` is omitted. * ``GitConfigParser`` uses Git's configuration syntax and canonical key spelling, and no longer subclasses ``configparser.RawConfigParser``. + Subsection names and values preserve undecodable bytes with ``surrogateescape``. Common getters, typed values, duplicate values, file/stream inputs and mutation methods remain, including ``GitConfigParser[section] = values`` for replacing nonempty sections. Invalid Git syntax previously accepted by Python is rejected. diff --git a/git/config.py b/git/config.py index 7219aae63..ca994f918 100644 --- a/git/config.py +++ b/git/config.py @@ -17,7 +17,7 @@ from typing import Any, Dict, Generic, Iterator, List, OrderedDict, Sequence, Tuple, TypeVar, Union, TYPE_CHECKING from typing import Mapping -from git.compat import defenc, force_text +from git.compat import safe_decode from git.exc import GitCommandError from git.types import _T, ConfigLevels_Tup, Lit_config_levels, PathLike, assert_never @@ -368,11 +368,11 @@ def read(self) -> None: if not record: continue key, separator, value = record.partition(b"\n") - prefix, option = key.decode(defenc).rsplit(".", 1) + prefix, option = safe_decode(key).rsplit(".", 1) section = self._public_section(prefix) if section not in sections: sections[section] = _OMD() - sections[section].add(option, value.decode(defenc) if separator else None) + sections[section].add(option, safe_decode(value) if separator else None) self._sections = sections self._is_initialized = True @@ -476,7 +476,7 @@ def _mutate(self, operation: str, *args: str) -> None: def _value(value: Any) -> str: if value is None: raise ValueError("Writing valueless Git configuration options is unsupported") - value = str(value) if isinstance(value, (int, float, bool)) else force_text(value) + value = str(value) if isinstance(value, (int, float, bool)) else safe_decode(value) if any(c in value for c in "\0\r"): raise ValueError("Git configuration values must not contain CR or NUL") return value diff --git a/test/test_config.py b/test/test_config.py index 5b4984172..abf4df5c4 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -69,6 +69,42 @@ def test_config_values_and_mutations(config_directory, stream): assert (source.getvalue() if stream else path.read_bytes()).startswith(b"[CoRe]\nFlag\n") +@pytest.mark.parametrize("stream", [False, True]) +def test_config_preserves_non_utf8_subsections_and_values(stream): + path = read_only_fixture("config-bytes.sh") + source = BytesIO((path / "config").read_bytes()) if stream else path / "config" + assert ( + Git() + .config("list", "--file", str(path / "config"), "--null", stdout_as_string=False) + .endswith(b"remote.origin-\xff.url\ntarget-\xfe\0") + ) + with GitConfigParser(source) as config: + assert 'remote "origin-\udcff"' in config.sections() + assert config.get('remote "origin-\udcff"', "url") == "target-\udcfe" + with Repo(path / "repo") as repo: + remote = repo.remotes[0] + assert remote.name == "origin-\udcff" + assert remote.url == "target-\udcfe" + + +@pytest.mark.skipif(os.name == "nt", reason="Windows command arguments cannot carry arbitrary bytes") +@pytest.mark.parametrize("stream", [False, True]) +def test_config_mutations_preserve_non_utf8_values(stream): + with writable_fixture("config-bytes.sh") as fixture: + path = fixture.path() / "config" + source = BytesIO(path.read_bytes()) if stream else path + with GitConfigParser(source, read_only=False) as config: + section = 'remote "origin-\udcff"' + assert config.get(section, "url") == "target-\udcfe" + config.set_value(section, "url", "updated-\udcfd") + config.add_value(section, "url", b"bytes-\xfc") + assert config.get_values(section, "url") == ["updated-\udcfd", "bytes-\udcfc"] + content = source.getvalue() if stream else path.read_bytes() + assert b'"origin-\xff"' in content + assert b"updated-\xfd" in content + assert b"bytes-\xfc" in content + + def test_config_includes_and_repository_conditions(): path = read_only_fixture("config.sh", "includes") with Repo(path / "repo") as repo: diff --git a/tests/fixtures/config-bytes.sh b/tests/fixtures/config-bytes.sh new file mode 100644 index 000000000..1f1f0bb8d --- /dev/null +++ b/tests/fixtures/config-bytes.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -eu +git init -q repo +printf '[remote "origin-\377"]\n\turl = target-\376\n' >> repo/.git/config +cp repo/.git/config config From c21f308b0edb5d26b6a94aa3751a4e3c7be85e54 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:15:05 +0100 Subject: [PATCH 17/21] fix(gix): preserve bytes in native submodule configuration values The dedicated Gix submodule parser returns path, URL and branch values as bytes. Strict decoding forced valid non-UTF-8 values into the CLI fallback. Decode those values with the shared lossless helper while preserving existing checks for duplicate sections, implicit values and raw branch spelling. Record the remaining native API gap: non-UTF-8 subsection names require CLI because generic config keys require text, and the byte-capable branch getter interprets empty/implicit values instead of exposing their raw spelling. No Python configuration semantics or fabricated native success are introduced. The native regression fails before the fix and passes with Git.execute blocked. Scripted tests also verify byte subsection names through fallback and cached field reads against Git 2.54.0. Connected tests pass: nine Gix cases, two CLI cases and seven native-only skips. Ruff, mypy and whitespace checks pass. --- doc/gix-backend.md | 12 +++++++++++- git/_backend.py | 2 +- test/test_submodule.py | 29 +++++++++++++++++++++++++++++ tests/fixtures/submodule-bytes.sh | 16 ++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/submodule-bytes.sh diff --git a/doc/gix-backend.md b/doc/gix-backend.md index 172db6858..3a680f563 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -369,7 +369,7 @@ and predate the current fallback; they do not establish native coverage. | Managed `symbolic_ref` | Nonrecursive target lookup | Mutation, command-level missing-reference diagnostics | | Reference enumeration, `for_each_ref` | Sorted reference names and literal prefixes, preserving symbolic aliases | Dangling symbolic refs, root refs, glob patterns, other formats/options; GIX-25 | | Managed `config --get KEY` | Merged repository config snapshot | Files/streams, enumeration, mutation; GIX-12 | -| Submodule enumeration and cached fields | Dedicated native `.gitmodules` parser over the existing worktree/blob source, with raw path/URL/branch values | Other/duplicate sections, ambiguous brackets, missing/implicit fields and parser errors; GIX-12 | +| Submodule enumeration and cached fields | Dedicated native `.gitmodules` parser over the existing worktree/blob source, with raw path/URL/branch values | Non-UTF-8 subsection names, other/duplicate sections, ambiguous brackets, missing/implicit fields and parser errors; GIX-12 | | Commit hooks | No bound native lookup/execution API | All hooks, including absent hooks; GIX-23 | | Worktree inventory, `worktree list` | Main and linked worktree metadata, including bare main repositories and locks | Prunable entries; GIX-14 | | `Repo.merge_base`, `Repo.is_ancestor` | Native graph queries for two revisions | Octopus/fork-point and other options | @@ -1088,6 +1088,16 @@ runtime validation and timings remain attributed to their measured snapshots. ### Remaining configuration reads (GIX-12) +Native `.gitmodules` reads preserve arbitrary bytes in paths, URLs and branch +values through `surrogateescape`. Non-UTF-8 subsection names use the CLI fallback: +GixPython 0.3.0's `ConfigFile.string()` and `.boolean()` require text keys. +Although `ModulesFile.path()` and `.url()` accept byte names, `.branch()` +interprets the value: an empty value becomes `HEAD`, and missing and implicit +values both become `None`. A byte-key raw configuration accessor is needed to +retain GitPython's configuration contract natively. The scripted +`submodule-bytes.sh` regressions cover values, names and cached-field rereads; +the native value test prohibits CLI execution. + The October 7 inventory identified 951 remote-enumeration reads, 674 actor lookups, 262 remote-property reads, 202 tracking-branch lookups and 111 fetch refspec checks. These are historical call-site counts, not additive savings. diff --git a/git/_backend.py b/git/_backend.py index 6fbb3fe94..b6109d46f 100644 --- a/git/_backend.py +++ b/git/_backend.py @@ -425,7 +425,7 @@ def submodule_config(source: Any) -> Any: if option != "branch" or config.boolean(key) is not None: raise _Unsupported("missing or implicit submodule fields") else: - values[option] = value.decode(defenc) + values[option] = safe_decode(value) result[name] = values except _Unsupported as exc: return _fallback("Submodule.config", str(exc)) diff --git a/test/test_submodule.py b/test/test_submodule.py index 335f2067d..af7e1fdd3 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -2,6 +2,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc +from io import BytesIO import os import os.path as osp from pathlib import Path @@ -32,6 +33,34 @@ from test.lib.fixtures import fixture_repo, read_only_fixture, writable_fixture +@pytest.mark.parametrize("scenario, name", [("ascii-name", "module"), ("byte-name", "module-\udcff")]) +def test_submodule_configuration_preserves_non_utf8_bytes(scenario, name): + with fixture_repo("submodule-bytes.sh", scenario) as repo: + data = repo.git.config("list", "--null", "--file", ".gitmodules", stdout_as_string=False) + assert b".path\npath-\xfe\0" in data + assert b".url\nurl-\xfd\0" in data + assert b".branch\nrefs/heads/branch-\xfc\0" in data + submodule = repo.submodules[0] + assert submodule.name == name + assert submodule.path == "path-\udcfe" + assert submodule.url == "url-\udcfd" + assert submodule.branch_path == "refs/heads/branch-\udcfc" + submodule._clear_cache() + assert (submodule.path, submodule.url, submodule.branch_path) == ( + "path-\udcfe", + "url-\udcfd", + "refs/heads/branch-\udcfc", + ) + + +@pytest.mark.skipif(git._backend.name != "gix", reason="requires the native backend") +def test_native_submodule_configuration_preserves_non_utf8_values(): + path = read_only_fixture("submodule-bytes.sh", "ascii-name") / "repo" / ".gitmodules" + with mock.patch.object(Git, "execute", side_effect=AssertionError("Git must not run")): + native = git._backend.submodule_config(BytesIO(path.read_bytes())) + assert native == {"module": {"path": "path-\udcfe", "url": "url-\udcfd", "branch": "refs/heads/branch-\udcfc"}} + + @pytest.mark.parametrize("state", ["unborn", "untracked", "detached"]) @pytest.mark.parametrize("remote", [None, ".", "origin"], ids=["working-tree", "local-origin", "origin"]) @patch_git_config("protocol.file.allow", "always") diff --git a/tests/fixtures/submodule-bytes.sh b/tests/fixtures/submodule-bytes.sh new file mode 100644 index 000000000..1ceaeb9d4 --- /dev/null +++ b/tests/fixtures/submodule-bytes.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -eu +git init -q repo +cd repo +git -c user.name=fixture -c user.email=fixture@example.invalid commit --allow-empty -qm initial +case "$1" in + ascii-name) name=module ;; + byte-name) name=$'module-\377' ;; + *) exit 1 ;; +esac +printf '[submodule "%s"]\npath = path-\376\nurl = url-\375\nbranch = refs/heads/branch-\374\n' "$name" > .gitmodules +modules=$(git hash-object -w .gitmodules) +commit=$(git rev-parse HEAD) +tree=$(printf '100644 blob %s\t.gitmodules\000160000 commit %s\tpath-\376\000' "$modules" "$commit" | git mktree -z) +commit=$(printf 'Add submodule\n' | git -c user.name=fixture -c user.email=fixture@example.invalid commit-tree "$tree" -p "$commit") +git update-ref HEAD "$commit" From bafa167e9aa5b6e18967bce6956fcef948bcee49 Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:15:28 +0100 Subject: [PATCH 18/21] fix(stats): return filenames instead of Git-quoted spellings `Commit.stats.files` exposed Git's C-quoted output as dictionary keys. With `core.quotePath=true`, byte names and many Unicode names were quoted; tabs, newlines, quotes and backslashes could also produce keys that were not paths. After parsing the numeric records, unquote filename keys with the existing diff path decoder and lossless byte/text helpers. Unquoting after framing keeps tabs and newlines inside filenames from becoming record delimiters. Native statistics and the existing Gix filename-quoting fallback are unchanged. The scripted Git 2.54.0 regression covers both quotePath settings, byte and Unicode names, literal quoting/backslashes, root/update/delete commits and binary counts. All eight subtests pass with both backends after failing before the fix; the connected CLI stats/rename selection passes five tests and eight subtests. Ruff, mypy and whitespace checks pass. --- doc/source/changes.rst | 3 +- git/objects/commit.py | 14 +++++-- test/test_commit.py | 47 +++++++++++++++++++++++ tests/fixtures/commit-stats-byte-paths.sh | 27 +++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/commit-stats-byte-paths.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index dbccd79ea..d94d2f2c4 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -69,7 +69,8 @@ API changes * ``Commit.stats`` preserves Unicode line separators in filenames when ``core.quotePath=false``. Git's LF-delimited output remains intact for root, subsequent and empty commits, including the Gix backend's filename - quoting fallback. + quoting fallback. File keys contain the actual names, unquoting Git's C-style + escapes and preserving undecodable bytes with ``surrogateescape``. * Imports from a source checkout now read ``git.__version__`` from ``VERSION``. Built wheels and source distributions continue to carry their stamped release version. diff --git a/git/objects/commit.py b/git/objects/commit.py index ab2fddfa3..dfc51b2f3 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -17,8 +17,9 @@ from git import _backend from git.cmd import Git +from git.compat import safe_decode, safe_encode from git.exc import BadObject, GitCommandError, UnsafeOptionError -from git.diff import Diffable +from git.diff import Diffable, decode_path from git.util import Actor, Stats, finalize_process, hex_to_bin from . import base @@ -47,7 +48,7 @@ cast, ) -from git.types import PathLike +from git.types import Files_TD, PathLike if TYPE_CHECKING: from git.refs import SymbolicReference @@ -529,7 +530,14 @@ def process_lines(lines: List[str]) -> str: "diff", self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True, raw=True ).split("\n") text = process_lines(lines) - return Stats._list_from_string(self.repo, text) + stats = Stats._list_from_string(self.repo, text) + # Unquote after parsing: filenames themselves may contain tabs and newlines. + files: Dict[PathLike, Files_TD] = { + safe_decode(cast(bytes, decode_path(safe_encode(os.fspath(path)), has_ab_prefix=False))): values + for path, values in stats.files.items() + } + stats.files = files + return stats @property def trailers_list(self) -> List[Tuple[str, str]]: diff --git a/test/test_commit.py b/test/test_commit.py index 99105c34d..06a0ff60c 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -15,6 +15,7 @@ from gitdb import IStream from git import Actor, Commit, Repo, _backend +from git.compat import safe_decode, safe_encode from git.exc import UnsafeOptionError from git.objects.util import tzoffset, utc @@ -225,6 +226,52 @@ def test_stats_preserve_unicode_line_separators(self): assert empty_stats.files == {} assert empty_stats.total == {"insertions": 0, "deletions": 0, "lines": 0, "files": 0} + def test_stats_preserve_quoted_filename_bytes(self): + paths = ( + b"byte-\xff", + b'"byte-\\377"', + "utf8-é".encode(), + b"tab\tname", + b"line\nname", + b'quote"name', + b"back\\slash", + ) + for quote_path in ("true", "false"): + with self.subTest(quote_path=quote_path), fixture_repo("commit-stats-byte-paths.sh", quote_path) as repo: + for revision, kind, added, deleted in ( + ("update-1", "A", 1, 0), + ("update-2", "M", 1, 0), + ("HEAD", "D", 0, 2), + ): + with self.subTest(revision=revision): + stats = repo.commit(revision).stats + expected = { + safe_decode(path): { + "insertions": added, + "deletions": deleted, + "lines": added + deleted, + "change_type": kind, + } + for path in paths + } + if kind != "M": + expected["binary\tfile"] = { + "insertions": 0, + "deletions": 0, + "lines": 0, + "change_type": kind, + } + assert stats.files == expected + assert {safe_encode(path) for path in stats.files} == set(paths) | ( + {b"binary\tfile"} if kind != "M" else set() + ) + assert stats.total == { + "insertions": len(paths) * added, + "deletions": len(paths) * deleted, + "lines": len(paths) * (added + deleted), + "files": len(expected), + } + def test_renames(self): commit = self.rorepo.commit("185d847ec7647fd2642a82d9205fb3d07ea71715") files = commit.stats.files diff --git a/tests/fixtures/commit-stats-byte-paths.sh b/tests/fixtures/commit-stats-byte-paths.sh new file mode 100644 index 000000000..19de12351 --- /dev/null +++ b/tests/fixtures/commit-stats-byte-paths.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main repo +cd repo +git config core.quotePath "$1" +git config diff.algorithm myers +# Tree entries avoid filesystem restrictions on byte and control-character paths. +paths=($'byte-\377' '"byte-\377"' $'utf8-\303\251' $'tab\tname' $'line\nname' 'quote"name' 'back\slash') +binary=$(printf 'binary\0data' | git hash-object -w --stdin) +for multiplier in 1 2; do + blob=$(for ((line=0; line Date: Sun, 25 Oct 2026 17:15:57 +0100 Subject: [PATCH 19/21] fix(refs): preserve Unicode whitespace in command records Git separates reference and remote URL records with LF, but Unicode `splitlines()` also split valid names at U+0085, U+2028 and U+2029. Enumeration could return a truncated, nonexistent reference. Fetch parsing and progress handling also stripped valid Unicode whitespace from the ends of names. Split these outputs only at LF, trim fetch column padding as ASCII spaces, and remove only CR/LF framing from progress records. Preserve empty remote-URL output behavior. Reference storage and native lookup remain owned by Git/Gix. Scripted SHA-1/SHA-256 repositories contain leading, middle and trailing separators in branch, tag and remote-ref names. Tests compare Git's inventory, resolve the names, feed bytes/text through the fetch progress handler, check local stale refs and retain Unicode separators in URLs. All 26 focused cases pass on each backend after reproducing the failures. Existing progress tests, Ruff, mypy and whitespace checks pass. --- doc/source/changes.rst | 2 + git/refs/symbolic.py | 2 +- git/remote.py | 12 +++--- git/util.py | 2 +- test/test_refs.py | 56 ++++++++++++++++++++++++++- tests/fixtures/ref-line-separators.sh | 17 ++++++++ 6 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/ref-line-separators.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d94d2f2c4..58c4cbfba 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -54,6 +54,8 @@ API changes * Packed tag names containing non-UTF-8 bytes are represented losslessly as surrogate-escaped strings. ``safe_encode()`` recovers their original bytes, and Gix reference validation and resolution receive those bytes directly. + Reference enumeration, fetch results, stale references and remote URLs preserve + Unicode line separators within names instead of treating them as record breaks. * Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``, and raw rename fields. Staged-file diffs no longer fail while decoding legacy filenames; blob contents remain byte streams regardless of their encoding. diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index d7d811fb3..68552f672 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -698,7 +698,7 @@ def _iter_items( output = repo.git._call_process_safe( "for_each_ref", "--format=%(refname)", *options, "--", *([prefix] if prefix else []) ) - for path in sorted(output.splitlines()): + for path in sorted(output.split("\n")): try: yield cls.from_path(repo, path) except ValueError: diff --git a/git/remote.py b/git/remote.py index 38094beb4..a122b1a5a 100644 --- a/git/remote.py +++ b/git/remote.py @@ -380,7 +380,7 @@ def _from_line(cls, repo: "Repo", line: str, refspecs: Sequence[str] = (), refs: if match is None: raise ValueError("Failed to parse fetch output: %r" % line) control, operation, source, destination, note = match.groups() - source = source.strip() + source = source.strip(" ") if control not in cls._flag_map: raise ValueError("Unknown fetch status: %r" % control) flags = cls._flag_map[cast(flagKeyLiteral, control)] @@ -393,7 +393,7 @@ def _from_line(cls, repo: "Repo", line: str, refspecs: Sequence[str] = (), refs: old_commit = None if control in (" ", "+"): old_commit = repo.rev_parse(operation.split("..", 1)[0]) - destination = destination.strip() + destination = destination.strip(" ") if destination == "FETCH_HEAD": ref = SymbolicReference(repo, destination) else: @@ -618,7 +618,9 @@ def delete_url(self, url: str, **kwargs: Any) -> "Remote": @property def urls(self) -> Iterator[str]: """:return: Iterator yielding all configured URL targets on a remote as strings""" - yield from self.repo.git._call_process_safe("remote", "get-url", "--all", "--", self.name).splitlines() + output = self.repo.git._call_process_safe("remote", "get-url", "--all", "--", self.name) + if output: + yield from output.split("\n") @property def refs(self) -> IterableList[RemoteReference]: @@ -655,7 +657,7 @@ def stale_refs(self) -> IterableList[Reference]: out_refs: IterableList[Reference] = IterableList(RemoteReference._id_attribute_, "%s/" % self.name) for line in self.repo.git._call_process_safe( "remote", "prune", "--dry-run", "--", self, _allow_network=True - ).splitlines()[2:]: + ).split("\n")[2:]: # expecting # * [would prune] origin/new_branch token = " * [would prune] " @@ -817,7 +819,7 @@ def _get_fetch_info_from_stderr( fetch_info_lines.append(line) continue - refs = self.repo.git._call_process_safe("for_each_ref", "--format=%(refname)").splitlines() + refs = self.repo.git._call_process_safe("for_each_ref", "--format=%(refname)").split("\n") try: configured = self.config_reader.config.get_values(self._config_section_name(), "fetch") except (cp.NoSectionError, cp.NoOptionError): diff --git a/git/util.py b/git/util.py index 8d6a57530..a9add1728 100644 --- a/git/util.py +++ b/git/util.py @@ -812,7 +812,7 @@ def new_message_handler(self) -> Callable[[str], None]: """ def handler(line: AnyStr) -> None: - self._parse_progress_line(line.rstrip()) + self._parse_progress_line(line.rstrip(b"\r\n") if isinstance(line, bytes) else line.rstrip("\r\n")) # END handler diff --git a/test/test_refs.py b/test/test_refs.py index b47fe40e5..1e47101c1 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -9,7 +9,7 @@ import unittest from itertools import chain from pathlib import Path -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -24,6 +24,8 @@ Head, Reference, RefLog, + Remote, + RemoteProgress, RemoteReference, SymbolicReference, TagReference, @@ -37,6 +39,58 @@ from test.lib.fixtures import fixture_repo, read_only_fixture, writable_fixture +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_reference_names_preserve_unicode_line_separators(object_format): + names = [ + before + separator + after + for separator in ("\x85", "\u2028", "\u2029") + for before, after in (("", "after"), ("before", "after"), ("before", "")) + ] + with fixture_repo("ref-line-separators.sh", object_format) as repo: + expected = { + f"{prefix}/{name}" for prefix in ("refs/heads", "refs/tags", "refs/remotes/origin") for name in names + } | {"refs/heads/main"} + assert set(repo.git.for_each_ref("--format=%(refname)").split("\n")) == expected + assert {ref.path for ref in repo.references} == expected + assert {head.name for head in repo.heads} == set(names) | {"main"} + assert {tag.name for tag in repo.tags} == set(names) + assert {ref.remote_head for ref in RemoteReference.iter_items(repo, remote="origin")} == set(names) + for ref in repo.references: + assert ref.commit == repo.head.commit + + +@pytest.mark.parametrize("separator", ["\x85", "\u2028", "\u2029"]) +@pytest.mark.parametrize("before,after", [("", "after"), ("before", "after"), ("before", "")]) +@pytest.mark.parametrize("as_bytes", [False, True]) +def test_fetch_resolves_unicode_line_separator_names_from_inventory(separator, before, after, as_bytes): + name = before + separator + after + with fixture_repo("ref-line-separators.sh", "sha1") as repo: + progress = RemoteProgress() + line = f" * [new branch] {name} -> origin/{name}" + progress.new_message_handler()((line + "\n").encode() if as_bytes else line + "\n") + assert progress.other_lines == [line] + with patch("git.remote.handle_process_output"): + result = Remote(repo, "inventory")._get_fetch_info_from_stderr(Mock(), progress) + assert len(result) == 1 + assert result[0].ref.path == f"refs/remotes/origin/{name}" + assert result[0].remote_ref_path == name + + +def test_remote_urls_preserve_unicode_line_separators(): + with fixture_repo("ref-line-separators.sh", "sha1") as repo: + assert list(repo.remote("unicode").urls) == [ + f"https://example.invalid/before{separator}after" for separator in ("\x85", "\u2028", "\u2029") + ] + with patch.object(Git, "_call_process_safe", return_value=""): + assert list(Remote(repo, "empty").urls) == [] + + +def test_stale_refs_preserve_unicode_line_separators(): + with fixture_repo("ref-line-separators.sh", "sha1") as repo: + origin = repo.remote("origin") + assert {ref.path for ref in origin.stale_refs} == {ref.path for ref in origin.refs} + + @pytest.mark.parametrize("object_format", ["sha1", "sha256"]) def test_non_utf8_tag_names_preserve_bytes(object_format): with fixture_repo("legacy-refs.sh", object_format) as repo: diff --git a/tests/fixtures/ref-line-separators.sh b/tests/fixtures/ref-line-separators.sh new file mode 100644 index 000000000..1504b1d4c --- /dev/null +++ b/tests/fixtures/ref-line-separators.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main --object-format="$1" repo +git init --bare --initial-branch=main --object-format="$1" upstream +cd repo +git remote add origin ../upstream +tree=$(git mktree Date: Sun, 25 Oct 2026 17:17:07 +0100 Subject: [PATCH 20/21] fix(gix): pass filename byte patterns to status and directory walks Status and untracked-file filtering passed surrogateescaped strings into GixPython. Converting those strings to Rust text raised `UnicodeEncodeError`, while equivalent Git CLI queries correctly matched the original byte names. Encode patterns with the shared lossless helper at the two native boundaries. `StatusPlatform.into_iter()` and `Repository.dirwalk_iter()` already accept byte patterns, so no new API, fallback or path matching implementation is needed. The scripted SHA-1/SHA-256 regressions fail before the fix and pass on both backends. They check matching deleted paths, nonmatching paths and a byte-bearing character class matching an untracked file. Gix checks prohibit CLI execution. Mypy and Ruff pass. Windows operand tests remain excluded because equivalent arbitrary-byte CLI arguments cannot be represented there. --- doc/gix-backend.md | 4 ++++ git/_backend.py | 4 ++-- test/test_byte_pathspecs.py | 28 ++++++++++++++++++++++++++++ tests/fixtures/pathspec-bytes.sh | 9 +++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 test/test_byte_pathspecs.py create mode 100644 tests/fixtures/pathspec-bytes.sh diff --git a/doc/gix-backend.md b/doc/gix-backend.md index 3a680f563..3d2f39238 100644 --- a/doc/gix-backend.md +++ b/doc/gix-backend.md @@ -395,6 +395,10 @@ and predate the current fallback; they do not establish native coverage. Reference names and revisions are encoded with `surrogateescape` before calling GixPython's byte-capable `Target.Symbolic`, `Repository.try_find_reference`, `Repository.rev_parse_single`, `Repository.rev_parse` and `RefEdit` APIs. +Status and untracked-file filters likewise pass byte patterns to +`StatusPlatform.into_iter()` and `Repository.dirwalk_iter()`. Their regression +checks matching and nonmatching patterns containing undecodable bytes, and +prohibits CLI fallback in Gix mode. This preserves non-UTF-8 packed tag names without interpreting reference storage in Python. `test_non_utf8_tag_names_preserve_bytes` covers lightweight and annotated tag enumeration in SHA-1 and SHA-256 repositories on both backends and all platforms. diff --git a/git/_backend.py b/git/_backend.py index b6109d46f..556b615c7 100644 --- a/git/_backend.py +++ b/git/_backend.py @@ -515,7 +515,7 @@ def is_dirty(command: Any, index: bool, working_tree: bool, untracked: bool, sub .index_worktree_submodules("configured" if submodules else "all", check_dirty=submodules) ) dirty = False - with status.into_iter(patterns) as entries: + with status.into_iter([safe_encode(pattern) for pattern in patterns]) as entries: for item in entries: if item.summary() is None: continue @@ -556,7 +556,7 @@ def untracked_files(command: Any, args: Tuple[Any, ...], options: Dict[str, Any] raise _Unsupported("invalid pathspec") walk_options = repo.dirwalk_options().emit_untracked("matching").emit_tracked(False) result = [] - with repo.dirwalk_iter(index, patterns, walk_options) as entries: + with repo.dirwalk_iter(index, [safe_encode(pattern) for pattern in patterns], walk_options) as entries: for item in entries: entry = item.entry if entry.status == "Untracked": diff --git a/test/test_byte_pathspecs.py b/test/test_byte_pathspecs.py new file mode 100644 index 000000000..2b2ea3747 --- /dev/null +++ b/test/test_byte_pathspecs.py @@ -0,0 +1,28 @@ +"""Path filters preserve Git's filename bytes at the native boundary.""" + +import contextlib +import os +from unittest.mock import patch + +import pytest + +from git import Git, _backend +from git.compat import safe_decode +from test.lib.fixtures import fixture_repo + + +@pytest.mark.skipif(os.name == "nt", reason="Windows CLI arguments cannot express arbitrary filename bytes") +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_byte_pathspecs_preserve_matches(object_format): + with fixture_repo("pathspec-bytes.sh", object_format) as repo: + guard = ( + patch.object(Git, "execute", side_effect=AssertionError("unexpected pathspec CLI fallback")) + if _backend.name == "gix" + else contextlib.nullcontext() + ) + with guard: + # The tracked path is absent in the worktree, so only its matching filter is dirty. + assert repo.is_dirty(path=safe_decode(b"tracked-\xff")) + assert not repo.is_dirty(path=safe_decode(b"absent-\xff")) + assert repo._get_untracked_files(safe_decode(b"[u\xff]*")) == ["untracked"] + assert repo._get_untracked_files(safe_decode(b"absent-\xff")) == [] diff --git a/tests/fixtures/pathspec-bytes.sh b/tests/fixtures/pathspec-bytes.sh new file mode 100644 index 000000000..13bfb2b04 --- /dev/null +++ b/tests/fixtures/pathspec-bytes.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main --object-format="$1" repo +cd repo +blob=$(printf 'contents\n' | git hash-object -w --stdin) +printf '100644 %s\ttracked-\377\0' "$blob" | git update-index -z --index-info +git commit -m initial +printf 'untracked\n' >untracked +printf 'other\n' >other From f5ca4a5865a51bfa293fd4587412994bc3fe2d5f Mon Sep 17 00:00:00 2001 From: Codex GPT-6 Date: Sun, 25 Oct 2026 17:18:42 +0100 Subject: [PATCH 21/21] fix(cmd): preserve bytes in streamed output and diagnostics Strict text decoding could make fetch/push output readers fail after Git successfully transferred refs, leaving empty result lists. Clone diagnostics, progress messages, failed-process stderr and timeout messages could likewise raise Unicode errors when names contained arbitrary bytes. Use the shared lossless helpers for output pumps, progress parsing and error re-encoding. Default subprocess text streams to surrogateescape while respecting an explicit errors policy. Keep binary streams binary and retain existing newline, timeout and process-finalization behavior. Nine regressions demonstrate the former failures and pass after the fix. They cover synchronous/streamed output, binary pumps, saved stderr, timeout and clone diagnostics, and actual SHA-1/SHA-256 fetch/push results. Reftable fixtures carry byte refs without invalid loose filenames or byte-valued refspec arguments. The connected Gix command/remote run passes 295 tests and four subtests; its remaining remote test passes with the project runner's isolated Git config instead of inherited fetch.prune=true. Ruff and mypy pass. --- doc/source/changes.rst | 3 + git/cmd.py | 17 +++--- git/util.py | 3 +- test/test_byte_output.py | 90 ++++++++++++++++++++++++++++ tests/fixtures/remote-byte-output.sh | 8 +++ 5 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 test/test_byte_output.py create mode 100644 tests/fixtures/remote-byte-output.sh diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 58c4cbfba..c8c71e9f8 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -47,6 +47,9 @@ Git executable and safety * Known command outcomes retain established GitPython/configparser exceptions where distinguishable. Other failures raise ``GitCommandError`` with Git's exit status and diagnostic output; exact error text may differ. + Text and streamed command output preserve undecodable bytes with + ``surrogateescape``, including fetch/push results, clone errors and timeout + diagnostics containing byte names. API changes ----------- diff --git a/git/cmd.py b/git/cmd.py index 254c661c8..ebef997ca 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -43,7 +43,7 @@ overload, ) -from git.compat import defenc, force_bytes, safe_decode +from git.compat import defenc, safe_decode, safe_encode from git import _backend from git.exc import ( CommandError, @@ -199,7 +199,7 @@ def pump_stream( if handler: if is_decode: assert isinstance(line, bytes) - line_str = line.decode(defenc) + line_str = safe_decode(line) handler(line_str) else: handler(line) @@ -451,7 +451,7 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: status = self.status p_stderr = None - stderr_b = force_bytes(data=stderr, encoding="utf-8") or b"" + stderr_b = safe_encode(stderr) or b"" if self._timeout_error is not None: stderr_b += (b"\n" if stderr_b else b"") + self._timeout_error.encode("utf-8") status = status or 1 @@ -459,7 +459,7 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: if stream: try: - return stderr_b + force_bytes(stream.read()) + return stderr_b + safe_encode(stream.read()) except (OSError, ValueError): return stderr_b or b"" else: @@ -1457,8 +1457,9 @@ def execute( If ``True``, default ``True``, we open stdout on the created process. :param universal_newlines: - If ``True``, pipes will be opened as text, and lines are split at all known - line endings. + If ``True``, pipes are opened as text with CR, LF and CRLF newline + handling. Undecodable bytes are preserved with ``surrogateescape`` + unless an explicit ``errors`` subprocess argument selects another policy. :param shell: Whether to invoke commands through a shell @@ -1569,6 +1570,8 @@ def execute( shell, universal_newlines, ) + if universal_newlines: + subprocess_kwargs.setdefault("errors", "surrogateescape") try: proc = safer_popen( command, @@ -1609,7 +1612,7 @@ def kill_process(pid: int) -> None: def make_timeout_error() -> Union[str, bytes]: err = f'Timeout: the command "{" ".join(redacted_command)}" did not complete in {timeout:g} secs.' - return err if universal_newlines else err.encode(defenc) + return err if universal_newlines else safe_encode(err) def communicate() -> Tuple[Union[str, bytes, None], Union[str, bytes, None]]: assert watchdog is not None diff --git a/git/util.py b/git/util.py index a9add1728..2d4cc7c23 100644 --- a/git/util.py +++ b/git/util.py @@ -85,6 +85,7 @@ from git.remote import Remote from git.repo.base import Repo +from git.compat import safe_decode from git.types import ( HSH_TD, Files_TD, @@ -728,7 +729,7 @@ def _parse_progress_line(self, line: AnyStr) -> Optional[object]: # Compressing objects: 100% (2/2) # Compressing objects: 100% (2/2), done. if isinstance(line, bytes): # mypy argues about ternary assignment. - line_str = line.decode("utf-8") + line_str = safe_decode(line) else: line_str = line self._cur_line = line_str diff --git a/test/test_byte_output.py b/test/test_byte_output.py new file mode 100644 index 000000000..5ab4de501 --- /dev/null +++ b/test/test_byte_output.py @@ -0,0 +1,90 @@ +"""Git output carries paths and reference names with arbitrary bytes.""" + +import os +import sys + +import pytest + +from git import Git, GitCommandError, Repo, RemoteProgress +from git.cmd import handle_process_output +from git.compat import safe_decode, safe_encode +from test.lib.fixtures import writable_fixture + + +@pytest.mark.parametrize("as_process", [False, True]) +def test_text_process_output_preserves_bytes(as_process): + proc = Git().execute( + [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'name-\\xff\\n')"], + universal_newlines=True, + as_process=as_process, + ) + if as_process: + lines = [] + handle_process_output(proc, lines.append, None, decode_streams=False) + proc.wait() + assert safe_encode("".join(lines)) == b"name-\xff\n" + else: + assert safe_encode(proc) == b"name-\xff" + + +def test_binary_process_output_handler_preserves_bytes(): + proc = Git().execute( + [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'name-\\xff\\n')"], as_process=True + ) + lines = [] + handle_process_output(proc, lines.append, None) + proc.wait() + assert safe_encode("".join(lines)) == b"name-\xff\n" + + +def test_process_error_preserves_previously_read_bytes(): + proc = Git().execute([sys.executable, "-c", "raise SystemExit(1)"], as_process=True) + with pytest.raises(GitCommandError) as error: + proc.wait(stderr=safe_decode(b"fatal: name-\xff")) + assert "name-\udcff" in str(error.value) + + +def test_progress_preserves_byte_diagnostics(): + progress = RemoteProgress() + progress._parse_progress_line(b"fatal: name-\xff") + assert safe_encode(progress.error_lines[0]) == b"fatal: name-\xff" + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX timeout and byte-argument handling") +def test_timeout_preserves_byte_arguments(): + with pytest.raises(GitCommandError) as error: + Git().execute( + [sys.executable, "-c", "import time; time.sleep(5)", safe_decode(b"name-\xff")], + kill_after_timeout=0.1, + ) + assert "name-\udcff" in str(error.value) + + +@pytest.mark.skipif(os.name == "nt", reason="Windows arguments cannot express arbitrary path bytes") +def test_clone_preserves_byte_path_in_failure(): + with writable_fixture("remote-byte-output.sh", "sha1") as temporary: + root = temporary.path() + with pytest.raises(GitCommandError) as error: + Repo.clone_from(root / safe_decode(b"missing-\xff"), root / "clone") + assert "missing-\udcff" in str(error.value) + + +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_fetch_and_push_preserve_byte_reference_results(object_format): + with writable_fixture("remote-byte-output.sh", object_format) as temporary: + root = temporary.path() + with Repo(root / "repo") as repo: + origin = repo.create_remote("origin", str(root / "source")) + fetched = origin.fetch() + assert {safe_encode(info.ref.path) for info in fetched} == { + b"refs/remotes/origin/main", + b"refs/remotes/origin/branch-\xff", + } + with Repo(root / "source") as repo: + target = repo.create_remote("target", str(root / "target")) + pushed = target.push("refs/heads/*:refs/heads/*") + assert not pushed.error + assert {safe_encode(info.remote_ref_string) for info in pushed} == { + b"refs/heads/main", + b"refs/heads/branch-\xff", + } diff --git a/tests/fixtures/remote-byte-output.sh b/tests/fixtures/remote-byte-output.sh new file mode 100644 index 000000000..5c163333b --- /dev/null +++ b/tests/fixtures/remote-byte-output.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eu +git init --initial-branch=main --object-format="$1" --ref-format=reftable source +git -C source commit --allow-empty -m initial +oid=$(git -C source rev-parse HEAD) +printf 'create refs/heads/branch-\377\0%s\0' "$oid" | git -C source update-ref --stdin -z +git init --initial-branch=main --object-format="$1" --ref-format=reftable repo +git init --bare --object-format="$1" --ref-format=reftable target