From ad54d9b487ca2741e5a3545ad1f773dbb057659e Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:46:11 +0700 Subject: [PATCH] Copy code entries instead of translating them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P07 says a doctest or a literal block is byte-identical to its msgid, because these are copied and never translated. 31 entries in the corpus were not, and the two ways they got there are both a layer being more forgiving than the one that checks it. 26 came in as human. sync.human_segments took any translated non-fuzzy entry from the mirror, and 136 of those are code. human is a provenance and not a grade: it says a person typed the string, and 30 of the 136 are a person having typed over the code. From tutorial/introduction.po: File "", line 1, in the English File "1", line 1, in 2 the translation and are gone. Elsewhere it is the indentation inside a for body flattened to one space, the carets under a syntax error no longer under what they point at, and a column-aligned option table reflowed to single spaces. Every one is an example a reader copies out and then has to debug. What they were made for is the comment translation, and that is worth less than the code: comments are M8, with a prompt of their own and a check that every code line came back byte-identical. The other 106 are already byte-identical and lose nothing by going. apply mints them from the msgid with passthrough=doctest on them, which is the same string with an accurate account of where it came from. 5 came in as machine, from a run made before the classifier could recognise them. batch filters non-prose out, so nothing queues one now, but the segments outlived the rule that let them be asked: python fibo.py was written into the corpus as python fibo.py <đối số>. apply now copies a code entry whatever the memory holds, so the guarantee does not depend on no stale segment existing. The classifier has got stricter twice and will again, and each time it does the memory acquires another handful of translations of things that turned out to be code. The human guard still comes first, so neither clause can quietly replace a reviewed string with the English. --refuzzy is the only way past it, which is how the 136 already in the corpus were rebuilt. load_human reconciles now rather than extending. An extend leaves all 136 in the memory for apply to write back, and the alternative was editing them out of the manifest by hand, which would make the memory something the content repo can no longer be rebuilt from. Dropping is safe because the mirror is the only place a human segment comes from: there is no command that promotes a string to human, deliberately, so anything of that provenance was read out of Transifex and can be read again. Machine segments are left alone, being the one thing here that cannot be rebuilt without spending the run. Only code is dropped, not everything the classifier calls non-prose. One no-op is a :ref: whose display text a person translated correctly, and that is a bug in is_noop rather than a licence to throw the translation away. Kind.code is one property where P07 and human_segments held a copy of the set each, and they disagreed about that :ref: entry. Measured over the corpus: P07 31 to 0, P05 1 to 0, six hard checks failing to four, 1 492 findings to 1 459. --- pyproject.toml | 2 +- src/pydocvi/apply.py | 23 +++++++++ src/pydocvi/audit/placeholders.py | 2 +- src/pydocvi/classify.py | 12 +++++ src/pydocvi/cli.py | 4 +- src/pydocvi/sync.py | 79 +++++++++++++++++++++++++++++-- tests/test_apply.py | 31 ++++++++++++ tests/test_classify.py | 10 ++++ tests/test_corpus.py | 23 ++++++++- tests/test_sync.py | 60 +++++++++++++++++++++++ 10 files changed, 236 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11538c2..e920261 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.5" +version = "0.1.6" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/apply.py b/src/pydocvi/apply.py index 815af63..8d73db2 100644 --- a/src/pydocvi/apply.py +++ b/src/pydocvi/apply.py @@ -269,6 +269,27 @@ def apply_entry( pipeline failing until the memory caught up with the catalog. What matters is that no machine string ever lands on top of a human one, and none does. + A code entry is copied whatever the memory holds for it, which is a rule + about the memory being wrong rather than empty. ``P07`` says a doctest is + byte-identical to its ``msgid``, and until this clause existed the only thing + standing behind that was nothing having queued one: :func:`batch` filters + non-prose out, so a segment for a code entry could only come from a run made + before the classifier learned to recognise it. Five did. ``python fibo.py + `` came back as ``python fibo.py <đối số>`` and was written into + the corpus, because the segment was there and this function had no reason to + doubt it. + + Refusing it here rather than pruning the memory is what makes the guarantee + hold going forward. The classifier will get stricter again, that is the + direction it has moved twice, and each time it does the memory acquires + another handful of translations of things that turned out to be code. What + they are worth is decided by what the string is now, not by what some earlier + run thought when it asked. + + The human guard still comes first, so this cannot quietly replace a reviewed + string with the English. Getting past that is ``--refuzzy`` and nothing else, + which is the same answer as everywhere else in this module. + An entry the memory has nothing for is either prose nobody has translated yet, which is left as upstream had it, or a no-op, which is copied through with the classifier's reason on it. See :func:`_copied`. @@ -285,6 +306,8 @@ def apply_entry( """ if existing is not None and is_human(existing) and not _contradicted(segment, refuzzy): return existing + if classify.classify(upstream.msgid).code: + return _copied(upstream) if segment is None or not segment.msgstr: return _copied(upstream) if not _translatable(upstream) else _untranslated(upstream) if segment.source == "human": diff --git a/src/pydocvi/audit/placeholders.py b/src/pydocvi/audit/placeholders.py index 1c37ca0..ab4f00b 100644 --- a/src/pydocvi/audit/placeholders.py +++ b/src/pydocvi/audit/placeholders.py @@ -228,7 +228,7 @@ def p07_code_is_byte_identical(corpus: Corpus) -> Iterator[Finding]: """ for one, entry in corpus.translated(): kind = classify.classify(entry.msgid) - if kind not in {classify.Kind.DOCTEST, classify.Kind.LITERAL_BLOCK}: + if not kind.code: continue if entry.msgstr != entry.msgid: yield Finding( diff --git a/src/pydocvi/classify.py b/src/pydocvi/classify.py index c63bfb2..512b900 100644 --- a/src/pydocvi/classify.py +++ b/src/pydocvi/classify.py @@ -91,6 +91,18 @@ class Kind(StrEnum): def translatable(self) -> bool: return self is Kind.PROSE + @property + def code(self) -> bool: + """Whether the entry is source text, to be copied and never written. + + A narrower claim than ``not translatable``. A no-op is not translatable + either, but it is markup, and the difference decides what happens to a + translation somebody has already made of one. ``P07`` reads this set and + so does :func:`sync.human_segments`, and they held a copy each until the + two disagreed about one entry. + """ + return self in {Kind.DOCTEST, Kind.LITERAL_BLOCK} + @dataclass(frozen=True, slots=True, kw_only=True) class Counts: diff --git a/src/pydocvi/cli.py b/src/pydocvi/cli.py index f0b3591..f320406 100644 --- a/src/pydocvi/cli.py +++ b/src/pydocvi/cli.py @@ -136,8 +136,8 @@ def sync_command( ) if human: - stored = sync.load_human(memory, catalogs) - console.print(f"human segments stored: {stored:,}") + loaded = sync.load_human(memory, catalogs) + console.print(f"human segments stored: {loaded.stored:,} dropped: {loaded.dropped:,}") if dry_run: console.print("[yellow]dry run, nothing written[/yellow]") diff --git a/src/pydocvi/sync.py b/src/pydocvi/sync.py index 5db96c9..b4d352e 100644 --- a/src/pydocvi/sync.py +++ b/src/pydocvi/sync.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from pathlib import Path -from pydocvi import catalog +from pydocvi import catalog, classify from pydocvi.catalog import Catalog from pydocvi.memory import Memory, Segment @@ -114,6 +114,19 @@ def quote(value: str) -> str: return f'"{value}"' +@dataclass(frozen=True, slots=True, kw_only=True) +class HumanLoad: + """What reconciling the memory against the mirror did. + + Both numbers, rather than the one ``load_human`` used to return. A run that + stores 0 and drops 136 and a run that does nothing are the same integer from + the caller's side, and the first is the one worth printing. + """ + + stored: int = 0 + dropped: int = 0 + + @dataclass(frozen=True, slots=True, kw_only=True) class SyncDiff: """What upstream has that the memory does not, and the other way round. @@ -218,18 +231,74 @@ def human_segments(catalogs: Iterable[Catalog]) -> list[Segment]: marked fuzzy. Fuzzy means gettext is not confident the translation still matches the source, and inheriting one of those as ground truth would seed the memory with the exact thing it exists to avoid. + + And when it is prose. A doctest is copied, never translated, which is the + rule ``P07`` enforces on everything this pipeline writes, and it was not + being asked of the 136 code entries the mirror hands over as somebody's + work. ``human`` is a provenance and not a grade: it says a person typed the + string, and 30 of those 136 are a person having typed over the code. + + What that looks like in the corpus, from ``tutorial/introduction.po``:: + + File "", line 1, in the English + File "1", line 1, in 2 the translation + + ```` and ```` are gone. Elsewhere it is the indentation + inside a ``for`` body flattened to one space, the carets under a syntax + error unaligned from what they point at, and a column-aligned option table + reflowed. Every one is an example a reader copies out and then has to + debug, and the comment translation they were made for is worth less than + that: comments are M8, with a prompt of their own and a check that every + code line came back byte-identical. + + The other 106 are already byte-identical, and dropping those loses nothing. + :func:`apply` mints them from the ``msgid`` with ``passthrough=doctest`` on + them, which is the same string with an accurate account of where it came + from instead of a claim that somebody translated it. + + Only code. A no-op is not translatable either and is left alone, because + one of them is a ``:ref:`` whose display text a person translated correctly + and the classifier calls markup. That entry is a bug in :func:`is_noop`, + not a licence to throw the translation away. """ out: list[Segment] = [] for cat in catalogs: for entry in cat: - if entry.translated and not entry.fuzzy: + if entry.translated and not entry.fuzzy and not classify.classify(entry.msgid).code: out.append(Segment.from_entry(entry, source="human")) return out -def load_human(memory: Memory, catalogs: Iterable[Catalog]) -> int: - """Load human translations into the memory. Returns how many were stored.""" - return memory.extend(human_segments(catalogs)) +def load_human(memory: Memory, catalogs: Iterable[Catalog]) -> HumanLoad: + """Bring the memory's human half into line with the mirror. + + Stores what the mirror offers and drops the ``human`` segments it no longer + does, which is a reconciliation where this used to be an ``extend``. + + The difference is only visible when :func:`human_segments` gets stricter, and + it got stricter once: 136 code entries stopped qualifying, and an ``extend`` + leaves all 136 sitting in the memory as somebody's translation of a doctest + for ``apply`` to write back. The alternative was editing them out of the + manifest by hand, and a memory that has been hand-edited is no longer a thing + the content repo can be rebuilt from, which is the property the whole + projection rests on. + + Dropping is safe because the mirror is the only place a ``human`` segment + comes from. There is no command in this tool that promotes a string to + ``human``, deliberately (spec 02 §4), so anything of that provenance in the + memory was read out of Transifex and can be read again. + + Only ``human``. A ``machine`` segment is the one thing here that genuinely + cannot be rebuilt without spending the run again, and nothing about the + mirror is evidence either way about it. + """ + wanted = human_segments(catalogs) + stored = memory.extend(wanted) + keep = {segment.id for segment in wanted} + dropped = [s.id for s in memory if s.source == "human" and s.id not in keep] + for one in dropped: + memory.remove(one) + return HumanLoad(stored=stored, dropped=len(dropped)) def diff(memory: Memory, catalogs: Iterable[Catalog]) -> SyncDiff: diff --git a/tests/test_apply.py b/tests/test_apply.py index c857be5..38ff39d 100644 --- a/tests/test_apply.py +++ b/tests/test_apply.py @@ -174,6 +174,37 @@ def test_the_memory_still_wins_over_the_copy(self) -> None: entry = entry_of(applied(cat, memory), "Added in version 3.9.") assert entry.msgstr == "Thêm vào phiên bản 3.9." + def test_a_segment_for_a_code_entry_is_refused(self) -> None: + """Where the memory wins over the copy stops. Five literal blocks in the + corpus had a ``gpt-5-6-mini`` segment from a run made before the + classifier could recognise them, and this function had no reason to doubt + it: ``python fibo.py `` was written into the corpus as ``python + fibo.py <đối số>``. What a string is worth is decided by what it is now, + not by what an earlier run thought when it asked.""" + code = "python fibo.py " + cat = upstream(block(code)) + memory = Memory([machine(code, "python fibo.py <đối số>")]) + entry = entry_of(applied(cat, memory), code) + assert entry.msgstr == code + assert entry.comments[-1] == "# pydocvi: passthrough=literal_block" + + def test_refusing_it_does_not_reach_past_the_reviewer(self) -> None: + """The human guard comes first, so this cannot quietly replace a reviewed + string with the English. ``--refuzzy`` is the only way past it, here as + everywhere else in this module.""" + code = ">>> n # try it\nNameError" + source = upstream(block(code.replace("\n", "\\n"))) + existing = upstream(block(code.replace("\n", "\\n"), "đã duyệt")) + out = apply.apply_catalog(source, existing, Memory(), stamp=STAMP)[0] + assert entry_of(out, code).msgstr == "đã duyệt" + + def test_refuzzy_reaches_it(self) -> None: + code = ">>> n # try it\nNameError" + source = upstream(block(code.replace("\n", "\\n"))) + existing = upstream(block(code.replace("\n", "\\n"), "đã duyệt")) + out = apply.apply_catalog(source, existing, Memory(), stamp=STAMP, refuzzy=True)[0] + assert entry_of(out, code).msgstr == code + def test_a_copy_is_not_counted_as_work_done(self) -> None: """Spec 12 §5: a no-op is neither translated nor outstanding. Folding 13 900 of them into the written column would report a doctest copied diff --git a/tests/test_classify.py b/tests/test_classify.py index 4be2d87..84179c1 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -194,6 +194,16 @@ def test_prose_is_the_only_translatable_kind(self) -> None: assert Kind.PROSE.translatable assert not any(kind.translatable for kind in Kind if kind is not Kind.PROSE) + def test_code_is_the_two_kinds_that_are_source_text(self) -> None: + assert {kind for kind in Kind if kind.code} == {Kind.DOCTEST, Kind.LITERAL_BLOCK} + + def test_code_is_narrower_than_not_translatable(self) -> None: + """The distinction the two readers of this property turned on. A no-op is + not translatable either, and it is markup rather than source text, so a + translation somebody made of one is kept where a doctest's is not.""" + assert not Kind.NOOP.translatable + assert not Kind.NOOP.code + def test_ordinary_prose(self) -> None: assert classify.classify("Return the sorted list.") is Kind.PROSE diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 3e02e72..490c5fa 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -26,6 +26,17 @@ EXPECTED_CHARACTERS = 12_526_506 EXPECTED_HUMAN = 1_435 +#: How many of those the memory takes, which stopped being the same number when +#: ``human_segments`` started asking whether the entry is code. The 136 in the +#: gap are doctests and literal blocks, and 30 of them are a person having typed +#: over the code rather than translated anything. The rest are already +#: byte-identical and ``apply`` mints them from the ``msgid`` instead. +#: +#: Two constants where there was one, on purpose. The mirror's count is a fact +#: about the mirror and this is a decision this tool makes, and folding them +#: back together would hide the next change to either. +EXPECTED_HUMAN_SEGMENTS = 1_299 + #: Re-measured after the identifier rule was narrowed to need a dot, an #: underscore or a digit. 3 193 entries left ``version_marker``: 3 164 to prose #: and 29 to ``noop``, the latter being single letters that no rule but the @@ -95,11 +106,21 @@ def test_segment_ids_are_unique_within_a_file(catalogs: list[catalog.Catalog]) - def test_human_translations_are_loaded_as_human(catalogs: list[catalog.Catalog]) -> None: segments = sync.human_segments(catalogs) - assert len(segments) == EXPECTED_HUMAN + assert len(segments) == EXPECTED_HUMAN_SEGMENTS assert {s.source for s in segments} == {"human"} assert all(s.msgstr for s in segments) +def test_no_code_entry_reaches_the_memory_as_somebody_s_translation( + catalogs: list[catalog.Catalog], +) -> None: + """The 136 the count above leaves behind. A doctest is copied and never + translated, so a ``human`` segment holding one is a person's edit of the + code sitting in the memory waiting to be written back over it.""" + segments = sync.human_segments(catalogs) + assert [s.msgid[:60] for s in segments if classify.classify(s.msgid).code] == [] + + def test_markup_protection_round_trips_on_every_msgid(catalogs: list[catalog.Catalog]) -> None: """The gate M2 exists for. A placeholder that does not come back out is a corrupted entry, and one corrupted entry is worse than a thousand untranslated diff --git a/tests/test_sync.py b/tests/test_sync.py index 1ff5733..b8b087b 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,6 +1,8 @@ from pathlib import Path +from conftest import catalog_of, entry from pydocvi import catalog, sync +from pydocvi.catalog import Catalog from pydocvi.memory import Memory @@ -46,6 +48,43 @@ def test_human_segments_skip_fuzzy_entries(data_dir: Path) -> None: assert sync.human_segments([fuzzed]) == [] +def reviewed(msgid: str, msgstr: str) -> Catalog: + """One catalog holding one entry a person signed off on.""" + return catalog_of(entry(msgid, msgstr, flags=())) + + +def test_human_segments_skip_code_however_translated_it_looks() -> None: + """A doctest is copied and never translated, which is ``P07``'s rule, and the + mirror hands over 136 code entries as somebody's work. 30 of those are a + person having typed over the code: ``File "", line 1, in `` + arrives as ``File "1", line 1, in 2``. ``human`` says who typed the string, + not that the string is right.""" + coded = reviewed( + '>>> n\n File "", line 1, in ', '>>> n\nFile "1", line 1, in 2' + ) + assert sync.human_segments([coded]) == [] + + +def test_human_segments_skip_code_that_was_copied_correctly_too() -> None: + """The other 106 lose nothing by going. ``apply`` mints them from the + ``msgid`` with ``passthrough=doctest`` on them, which is the same string with + an accurate account of where it came from.""" + same = ">>> len([1, 2])\n2" + assert sync.human_segments([reviewed(same, same)]) == [] + + +def test_human_segments_keep_a_no_op_a_person_translated() -> None: + """Only code is dropped, not everything the classifier calls non-prose. One + no-op in the corpus is a ``:ref:`` whose display text a person translated + correctly, and that is a bug in ``is_noop`` rather than a licence to throw + the translation away.""" + noop = reviewed( + ":ref:`Documentation on attributes `.", + ":ref:`Tài liệu về các thuộc tính `.", + ) + assert len(sync.human_segments([noop])) == 1 + + def test_diff_reports_upstream_strings_the_memory_lacks(data_dir: Path) -> None: cat = catalog.read(data_dir / "small.po") changes = sync.diff(Memory(), [cat]) @@ -54,6 +93,27 @@ def test_diff_reports_upstream_strings_the_memory_lacks(data_dir: Path) -> None: assert not changes.clean +def test_load_human_drops_a_stale_human_segment() -> None: + """The case this stopped being an ``extend`` for. A doctest the mirror once + offered as somebody's translation is still in the memory after the rule that + admitted it got stricter, and ``apply`` would write it back over the code.""" + doctest = entry('>>> n\n File "", line 1, in ', "", flags=()) + memory = Memory([sync.Segment.from_entry(doctest, source="human")]) + loaded = sync.load_human(memory, [catalog_of(entry("Return a list.", "Trả về.", flags=()))]) + assert loaded.stored == 1 + assert loaded.dropped == 1 + assert [s.msgid for s in memory] == ["Return a list."] + + +def test_load_human_leaves_machine_segments_where_they_are() -> None: + """A machine segment is the one thing here that cannot be rebuilt without + spending the run again, and the mirror is no evidence either way about it.""" + memory = Memory([sync.Segment(id="0" * 16, msgid="x", msgstr="y", source="machine")]) + loaded = sync.load_human(memory, [catalog_of(entry("Return a list.", "Trả về.", flags=()))]) + assert loaded.dropped == 0 + assert {s.source for s in memory} == {"machine", "human"} + + def test_diff_reports_orphans(data_dir: Path) -> None: cat = catalog.read(data_dir / "small.po") memory = Memory()