Copy code entries instead of translating them

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "python-docs-vi-translator"
-version = "0.1.5"
+version = "0.1.6"
 description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes"
 readme = "README.md"
 license = "MIT"

src/pydocvi/apply.py

@@ -269,6 +269,27 @@ def apply_entry(
     pipeline failing until the memory caught up with the catalog. What matters is
     that no machine string ever lands on top of a human one, and none does.
 
+    A code entry is copied whatever the memory holds for it, which is a rule
+    about the memory being wrong rather than empty. ``P07`` says a doctest is
+    byte-identical to its ``msgid``, and until this clause existed the only thing
+    standing behind that was nothing having queued one: :func:`batch` filters
+    non-prose out, so a segment for a code entry could only come from a run made
+    before the classifier learned to recognise it. Five did. ``python fibo.py
+    <arguments>`` came back as ``python fibo.py <đối số>`` and was written into
+    the corpus, because the segment was there and this function had no reason to
+    doubt it.
+
+    Refusing it here rather than pruning the memory is what makes the guarantee
+    hold going forward. The classifier will get stricter again, that is the
+    direction it has moved twice, and each time it does the memory acquires
+    another handful of translations of things that turned out to be code. What
+    they are worth is decided by what the string is now, not by what some earlier
+    run thought when it asked.
+
+    The human guard still comes first, so this cannot quietly replace a reviewed
+    string with the English. Getting past that is ``--refuzzy`` and nothing else,
+    which is the same answer as everywhere else in this module.
+
     An entry the memory has nothing for is either prose nobody has translated
     yet, which is left as upstream had it, or a no-op, which is copied through
     with the classifier's reason on it. See :func:`_copied`.
@@ -285,6 +306,8 @@ def apply_entry(
     """
     if existing is not None and is_human(existing) and not _contradicted(segment, refuzzy):
         return existing
+    if classify.classify(upstream.msgid).code:
+        return _copied(upstream)
     if segment is None or not segment.msgstr:
         return _copied(upstream) if not _translatable(upstream) else _untranslated(upstream)
     if segment.source == "human":

src/pydocvi/audit/placeholders.py

@@ -228,7 +228,7 @@ def p07_code_is_byte_identical(corpus: Corpus) -> Iterator[Finding]:
     """
     for one, entry in corpus.translated():
         kind = classify.classify(entry.msgid)
-        if kind not in {classify.Kind.DOCTEST, classify.Kind.LITERAL_BLOCK}:
+        if not kind.code:
             continue
         if entry.msgstr != entry.msgid:
             yield Finding(

src/pydocvi/classify.py

@@ -91,6 +91,18 @@ class Kind(StrEnum):
     def translatable(self) -> bool:
         return self is Kind.PROSE
 
+    @property
+    def code(self) -> bool:
+        """Whether the entry is source text, to be copied and never written.
+
+        A narrower claim than ``not translatable``. A no-op is not translatable
+        either, but it is markup, and the difference decides what happens to a
+        translation somebody has already made of one. ``P07`` reads this set and
+        so does :func:`sync.human_segments`, and they held a copy each until the
+        two disagreed about one entry.
+        """
+        return self in {Kind.DOCTEST, Kind.LITERAL_BLOCK}
+
 
 @dataclass(frozen=True, slots=True, kw_only=True)
 class Counts:

src/pydocvi/cli.py

@@ -136,8 +136,8 @@ def sync_command(
     )
 
     if human:
-        stored = sync.load_human(memory, catalogs)
-        console.print(f"human segments stored: {stored:,}")
+        loaded = sync.load_human(memory, catalogs)
+        console.print(f"human segments stored: {loaded.stored:,}   dropped: {loaded.dropped:,}")
 
     if dry_run:
         console.print("[yellow]dry run, nothing written[/yellow]")

src/pydocvi/sync.py

@@ -11,7 +11,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-from pydocvi import catalog
+from pydocvi import catalog, classify
 from pydocvi.catalog import Catalog
 from pydocvi.memory import Memory, Segment
 
@@ -114,6 +114,19 @@ def quote(value: str) -> str:
     return f'"{value}"'
 
 
+@dataclass(frozen=True, slots=True, kw_only=True)
+class HumanLoad:
+    """What reconciling the memory against the mirror did.
+
+    Both numbers, rather than the one ``load_human`` used to return. A run that
+    stores 0 and drops 136 and a run that does nothing are the same integer from
+    the caller's side, and the first is the one worth printing.
+    """
+
+    stored: int = 0
+    dropped: int = 0
+
+
 @dataclass(frozen=True, slots=True, kw_only=True)
 class SyncDiff:
     """What upstream has that the memory does not, and the other way round.
@@ -218,18 +231,74 @@ def human_segments(catalogs: Iterable[Catalog]) -> list[Segment]:
     marked fuzzy. Fuzzy means gettext is not confident the translation still
     matches the source, and inheriting one of those as ground truth would seed
     the memory with the exact thing it exists to avoid.
+
+    And when it is prose. A doctest is copied, never translated, which is the
+    rule ``P07`` enforces on everything this pipeline writes, and it was not
+    being asked of the 136 code entries the mirror hands over as somebody's
+    work. ``human`` is a provenance and not a grade: it says a person typed the
+    string, and 30 of those 136 are a person having typed over the code.
+
+    What that looks like in the corpus, from ``tutorial/introduction.po``::
+
+        File "<stdin>", line 1, in <module>     the English
+        File "1", line 1, in 2                  the translation
+
+    ``<stdin>`` and ``<module>`` are gone. Elsewhere it is the indentation
+    inside a ``for`` body flattened to one space, the carets under a syntax
+    error unaligned from what they point at, and a column-aligned option table
+    reflowed. Every one is an example a reader copies out and then has to
+    debug, and the comment translation they were made for is worth less than
+    that: comments are M8, with a prompt of their own and a check that every
+    code line came back byte-identical.
+
+    The other 106 are already byte-identical, and dropping those loses nothing.
+    :func:`apply` mints them from the ``msgid`` with ``passthrough=doctest`` on
+    them, which is the same string with an accurate account of where it came
+    from instead of a claim that somebody translated it.
+
+    Only code. A no-op is not translatable either and is left alone, because
+    one of them is a ``:ref:`` whose display text a person translated correctly
+    and the classifier calls markup. That entry is a bug in :func:`is_noop`,
+    not a licence to throw the translation away.
     """
     out: list[Segment] = []
     for cat in catalogs:
         for entry in cat:
-            if entry.translated and not entry.fuzzy:
+            if entry.translated and not entry.fuzzy and not classify.classify(entry.msgid).code:
                 out.append(Segment.from_entry(entry, source="human"))
     return out
 
 
-def load_human(memory: Memory, catalogs: Iterable[Catalog]) -> int:
-    """Load human translations into the memory. Returns how many were stored."""
-    return memory.extend(human_segments(catalogs))
+def load_human(memory: Memory, catalogs: Iterable[Catalog]) -> HumanLoad:
+    """Bring the memory's human half into line with the mirror.
+
+    Stores what the mirror offers and drops the ``human`` segments it no longer
+    does, which is a reconciliation where this used to be an ``extend``.
+
+    The difference is only visible when :func:`human_segments` gets stricter, and
+    it got stricter once: 136 code entries stopped qualifying, and an ``extend``
+    leaves all 136 sitting in the memory as somebody's translation of a doctest
+    for ``apply`` to write back. The alternative was editing them out of the
+    manifest by hand, and a memory that has been hand-edited is no longer a thing
+    the content repo can be rebuilt from, which is the property the whole
+    projection rests on.
+
+    Dropping is safe because the mirror is the only place a ``human`` segment
+    comes from. There is no command in this tool that promotes a string to
+    ``human``, deliberately (spec 02 §4), so anything of that provenance in the
+    memory was read out of Transifex and can be read again.
+
+    Only ``human``. A ``machine`` segment is the one thing here that genuinely
+    cannot be rebuilt without spending the run again, and nothing about the
+    mirror is evidence either way about it.
+    """
+    wanted = human_segments(catalogs)
+    stored = memory.extend(wanted)
+    keep = {segment.id for segment in wanted}
+    dropped = [s.id for s in memory if s.source == "human" and s.id not in keep]
+    for one in dropped:
+        memory.remove(one)
+    return HumanLoad(stored=stored, dropped=len(dropped))
 
 
 def diff(memory: Memory, catalogs: Iterable[Catalog]) -> SyncDiff:

tests/test_apply.py

@@ -174,6 +174,37 @@ def test_the_memory_still_wins_over_the_copy(self) -> None:
         entry = entry_of(applied(cat, memory), "Added in version 3.9.")
         assert entry.msgstr == "Thêm vào phiên bản 3.9."
 
+    def test_a_segment_for_a_code_entry_is_refused(self) -> None:
+        """Where the memory wins over the copy stops. Five literal blocks in the
+        corpus had a ``gpt-5-6-mini`` segment from a run made before the
+        classifier could recognise them, and this function had no reason to doubt
+        it: ``python fibo.py <arguments>`` was written into the corpus as ``python
+        fibo.py <đối số>``. What a string is worth is decided by what it is now,
+        not by what an earlier run thought when it asked."""
+        code = "python fibo.py <arguments>"
+        cat = upstream(block(code))
+        memory = Memory([machine(code, "python fibo.py <đối số>")])
+        entry = entry_of(applied(cat, memory), code)
+        assert entry.msgstr == code
+        assert entry.comments[-1] == "# pydocvi: passthrough=literal_block"
+
+    def test_refusing_it_does_not_reach_past_the_reviewer(self) -> None:
+        """The human guard comes first, so this cannot quietly replace a reviewed
+        string with the English. ``--refuzzy`` is the only way past it, here as
+        everywhere else in this module."""
+        code = ">>> n  # try it\nNameError"
+        source = upstream(block(code.replace("\n", "\\n")))
+        existing = upstream(block(code.replace("\n", "\\n"), "đã duyệt"))
+        out = apply.apply_catalog(source, existing, Memory(), stamp=STAMP)[0]
+        assert entry_of(out, code).msgstr == "đã duyệt"
+
+    def test_refuzzy_reaches_it(self) -> None:
+        code = ">>> n  # try it\nNameError"
+        source = upstream(block(code.replace("\n", "\\n")))
+        existing = upstream(block(code.replace("\n", "\\n"), "đã duyệt"))
+        out = apply.apply_catalog(source, existing, Memory(), stamp=STAMP, refuzzy=True)[0]
+        assert entry_of(out, code).msgstr == code
+
     def test_a_copy_is_not_counted_as_work_done(self) -> None:
         """Spec 12 §5: a no-op is neither translated nor outstanding. Folding
         13 900 of them into the written column would report a doctest copied

tests/test_classify.py

@@ -194,6 +194,16 @@ def test_prose_is_the_only_translatable_kind(self) -> None:
         assert Kind.PROSE.translatable
         assert not any(kind.translatable for kind in Kind if kind is not Kind.PROSE)
 
+    def test_code_is_the_two_kinds_that_are_source_text(self) -> None:
+        assert {kind for kind in Kind if kind.code} == {Kind.DOCTEST, Kind.LITERAL_BLOCK}
+
+    def test_code_is_narrower_than_not_translatable(self) -> None:
+        """The distinction the two readers of this property turned on. A no-op is
+        not translatable either, and it is markup rather than source text, so a
+        translation somebody made of one is kept where a doctest's is not."""
+        assert not Kind.NOOP.translatable
+        assert not Kind.NOOP.code
+
     def test_ordinary_prose(self) -> None:
         assert classify.classify("Return the sorted list.") is Kind.PROSE
 

tests/test_corpus.py

@@ -26,6 +26,17 @@
 EXPECTED_CHARACTERS = 12_526_506
 EXPECTED_HUMAN = 1_435
 
+#: How many of those the memory takes, which stopped being the same number when
+#: ``human_segments`` started asking whether the entry is code. The 136 in the
+#: gap are doctests and literal blocks, and 30 of them are a person having typed
+#: over the code rather than translated anything. The rest are already
+#: byte-identical and ``apply`` mints them from the ``msgid`` instead.
+#:
+#: Two constants where there was one, on purpose. The mirror's count is a fact
+#: about the mirror and this is a decision this tool makes, and folding them
+#: back together would hide the next change to either.
+EXPECTED_HUMAN_SEGMENTS = 1_299
+
 #: Re-measured after the identifier rule was narrowed to need a dot, an
 #: underscore or a digit. 3 193 entries left ``version_marker``: 3 164 to prose
 #: and 29 to ``noop``, the latter being single letters that no rule but the
@@ -95,11 +106,21 @@ def test_segment_ids_are_unique_within_a_file(catalogs: list[catalog.Catalog]) -
 
 def test_human_translations_are_loaded_as_human(catalogs: list[catalog.Catalog]) -> None:
     segments = sync.human_segments(catalogs)
-    assert len(segments) == EXPECTED_HUMAN
+    assert len(segments) == EXPECTED_HUMAN_SEGMENTS
     assert {s.source for s in segments} == {"human"}
     assert all(s.msgstr for s in segments)
 
 
+def test_no_code_entry_reaches_the_memory_as_somebody_s_translation(
+    catalogs: list[catalog.Catalog],
+) -> None:
+    """The 136 the count above leaves behind. A doctest is copied and never
+    translated, so a ``human`` segment holding one is a person's edit of the
+    code sitting in the memory waiting to be written back over it."""
+    segments = sync.human_segments(catalogs)
+    assert [s.msgid[:60] for s in segments if classify.classify(s.msgid).code] == []
+
+
 def test_markup_protection_round_trips_on_every_msgid(catalogs: list[catalog.Catalog]) -> None:
     """The gate M2 exists for. A placeholder that does not come back out is a
     corrupted entry, and one corrupted entry is worse than a thousand untranslated

tests/test_sync.py

@@ -1,6 +1,8 @@
 from pathlib import Path
 
+from conftest import catalog_of, entry
 from pydocvi import catalog, sync
+from pydocvi.catalog import Catalog
 from pydocvi.memory import Memory
 
 
@@ -46,6 +48,43 @@ def test_human_segments_skip_fuzzy_entries(data_dir: Path) -> None:
     assert sync.human_segments([fuzzed]) == []
 
 
+def reviewed(msgid: str, msgstr: str) -> Catalog:
+    """One catalog holding one entry a person signed off on."""
+    return catalog_of(entry(msgid, msgstr, flags=()))
+
+
+def test_human_segments_skip_code_however_translated_it_looks() -> None:
+    """A doctest is copied and never translated, which is ``P07``'s rule, and the
+    mirror hands over 136 code entries as somebody's work. 30 of those are a
+    person having typed over the code: ``File "<stdin>", line 1, in <module>``
+    arrives as ``File "1", line 1, in 2``. ``human`` says who typed the string,
+    not that the string is right."""
+    coded = reviewed(
+        '>>> n\n  File "<stdin>", line 1, in <module>', '>>> n\nFile "1", line 1, in 2'
+    )
+    assert sync.human_segments([coded]) == []
+
+
+def test_human_segments_skip_code_that_was_copied_correctly_too() -> None:
+    """The other 106 lose nothing by going. ``apply`` mints them from the
+    ``msgid`` with ``passthrough=doctest`` on them, which is the same string with
+    an accurate account of where it came from."""
+    same = ">>> len([1, 2])\n2"
+    assert sync.human_segments([reviewed(same, same)]) == []
+
+
+def test_human_segments_keep_a_no_op_a_person_translated() -> None:
+    """Only code is dropped, not everything the classifier calls non-prose. One
+    no-op in the corpus is a ``:ref:`` whose display text a person translated
+    correctly, and that is a bug in ``is_noop`` rather than a licence to throw
+    the translation away."""
+    noop = reviewed(
+        ":ref:`Documentation on attributes <class-attrs>`.",
+        ":ref:`Tài liệu về các thuộc tính <class-attrs>`.",
+    )
+    assert len(sync.human_segments([noop])) == 1
+
+
 def test_diff_reports_upstream_strings_the_memory_lacks(data_dir: Path) -> None:
     cat = catalog.read(data_dir / "small.po")
     changes = sync.diff(Memory(), [cat])
@@ -54,6 +93,27 @@ def test_diff_reports_upstream_strings_the_memory_lacks(data_dir: Path) -> None:
     assert not changes.clean
 
 
+def test_load_human_drops_a_stale_human_segment() -> None:
+    """The case this stopped being an ``extend`` for. A doctest the mirror once
+    offered as somebody's translation is still in the memory after the rule that
+    admitted it got stricter, and ``apply`` would write it back over the code."""
+    doctest = entry('>>> n\n  File "<stdin>", line 1, in <module>', "", flags=())
+    memory = Memory([sync.Segment.from_entry(doctest, source="human")])
+    loaded = sync.load_human(memory, [catalog_of(entry("Return a list.", "Trả về.", flags=()))])
+    assert loaded.stored == 1
+    assert loaded.dropped == 1
+    assert [s.msgid for s in memory] == ["Return a list."]
+
+
+def test_load_human_leaves_machine_segments_where_they_are() -> None:
+    """A machine segment is the one thing here that cannot be rebuilt without
+    spending the run again, and the mirror is no evidence either way about it."""
+    memory = Memory([sync.Segment(id="0" * 16, msgid="x", msgstr="y", source="machine")])
+    loaded = sync.load_human(memory, [catalog_of(entry("Return a list.", "Trả về.", flags=()))])
+    assert loaded.dropped == 0
+    assert {s.source for s in memory} == {"machine", "human"}
+
+
 def test_diff_reports_orphans(data_dir: Path) -> None:
     cat = catalog.read(data_dir / "small.po")
     memory = Memory()