From 9f6767db3ec3df71ff96091c68e06f7c50f0c7b4 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:46:11 +0700 Subject: [PATCH] Stop calling a role with prose in it a no-op is_noop strips the markup and asks whether anything is left. A role written `:ref:`text `` has words a reader sees inside the backticks, and stripping the span takes them away with the target. `:ref:`Documentation on attributes and methods on classes `` came out a no-op. Seven English words that batch never queued and Corpus.prose never showed to a check, so no model was asked and nothing reported it. That is the failure the is_version_marker docstring describes, in a different rule. It surfaced only because one person had translated one of these by hand, correctly, which made it the one non-prose entry in the mirror whose translation was not a copy of its source. 130 entries in the corpus are this shape, and they carry :ref: 117 times, :rfc: 6, :term: 4, :doc: 3 and :option: 2. All 130 display strings were read: `A logging cookbook`, `Basic Tutorial`, `Configurable memory allocators`, `Explaining the object-oriented interface`. Prose, all of it. The words are counted between the spaces rather than by running the word pattern over the text. That is worth 136 entries and every one of them is wrong: `:c:member:`base_exec_prefix `` is one identifier, and a pattern looking for runs of letters finds three words in it. A display text of one word stays a no-op, which is 81 `:c:member:` entries naming a struct field, and it is the cautious direction: a missed entry stays as it is today, and a false positive sends an identifier to a model to be translated into something that links nowhere. prose 75 593 to 75 723, noop 5 721 to 5 591, batches 2 801 to 2 803. --- pyproject.toml | 2 +- src/pydocvi/classify.py | 59 ++++++++++++++++++++++++++++++++++++++++- tests/test_classify.py | 36 +++++++++++++++++++++++++ tests/test_corpus.py | 10 ++++--- tests/test_sync.py | 12 ++++++--- uv.lock | 2 +- 6 files changed, 111 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a7901e0..e535559 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.9" +version = "0.1.10" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/classify.py b/src/pydocvi/classify.py index b13591c..09532ab 100644 --- a/src/pydocvi/classify.py +++ b/src/pydocvi/classify.py @@ -89,6 +89,24 @@ #: A quoted string, whose insides are data and not prose. _QUOTED = re.compile(r"'[^']*'|\"[^\"]*\"") +#: The visible half of a role or a link written with an explicit target, which +#: is ``:ref:`text ``` and ``` `text `_ ```. One pattern for both +#: because the shape is the same and so is the question about it. ``P03`` and +#: ``P06`` read the other half, the part that must not be translated. +_DISPLAY = re.compile(r"`([^`<>]+?)\s*<[^>]*>`") + +#: A whole word, anchored, so that a token has to be a word rather than contain +#: one. ``base_exec_prefix`` is one identifier and not three words. +_WHOLE = re.compile(r"^[A-Za-z]{2,}$") + +#: Punctuation to take off a token before asking whether it is a word. +_TRIM = ".,;:()[]'\"" + +#: How many words of display text stop an entry being a no-op. One word is +#: usually an identifier, and see :func:`_displayed` for why erring towards +#: leaving those alone is the cautious direction. +_PROSE_WORDS = 2 + #: A terminal prompt, with the virtualenv name some transcripts put in front of #: it. One of these anywhere in an entry makes the whole entry a transcript, #: because the lines around a prompt are the output of the command in it. @@ -369,8 +387,47 @@ def is_noop(msgid: str) -> bool: or more ASCII letters remains. ``:mod:`os.path``` is a no-op. ``the :mod:`os.path` module`` is not, and the difference is the four English words a reader would otherwise meet in the middle of a Vietnamese page. + + Unless the markup itself carries prose, which is the case this rule missed + for as long as it existed. A role written ``:ref:`text ``` has words + a reader sees inside the backticks, and stripping the span takes them away + with the target. ``:ref:`Documentation on attributes and methods on classes + ``` came out a no-op, which meant :func:`batch` + never queued it and :meth:`Corpus.prose` never showed it to a check. Seven + English words, invisible from every angle that would have reported them. + + It only surfaced because one person had translated one of these by hand, + correctly, and that made it the one non-prose entry in the mirror whose + translation was not a copy of its source. 130 entries in the corpus are this + shape. + + See :func:`_displayed` for where the line is drawn, which is the same line + :func:`is_version_marker` draws and for the same reason. + """ + return not _WORD.search(strip_markup(msgid)) and not _displayed(msgid) + + +def _displayed(msgid: str) -> bool: + """Whether a role or link in the entry carries prose a reader sees. + + Two or more whole words, counted between the spaces rather than by running + :data:`_WORD` over the text. The difference is 136 entries and every one of + them is wrong: ``:c:member:`base_exec_prefix ``` + is one identifier, and a pattern looking for runs of letters finds ``base``, + ``exec`` and ``prefix`` in it and calls that a sentence. + + So a display text of one word stays a no-op, 81 entries of ``:c:member:`` + naming a struct field. That is the same call :func:`is_version_marker` makes + about a bare token and it is deliberately the cautious one here: a missed + entry stays as it is today, and a false positive sends an identifier to a + model to be translated into something that links nowhere. """ - return not _WORD.search(strip_markup(msgid)) + return any(_words(text) >= _PROSE_WORDS for text in _DISPLAY.findall(msgid)) + + +def _words(text: str) -> int: + """How many whole words the display text has, ignoring punctuation.""" + return sum(1 for token in text.split() if _WHOLE.match(token.strip(_TRIM))) def counts(msgids: list[str]) -> Counts: diff --git a/tests/test_classify.py b/tests/test_classify.py index 8c506f9..aab54bf 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -30,6 +30,42 @@ def test_entries_with_no_prose_left(self, msgid: str) -> None: def test_entries_with_prose_around_the_markup(self, msgid: str) -> None: assert not classify.is_noop(msgid) + @pytest.mark.parametrize( + "msgid", + [ + ":ref:`Documentation on attributes and methods on classes `.", + ":ref:`A logging cookbook `", + ":doc:`Graphical User Interfaces with Tk `", + ":term:`Filesystem encoding `", + "`Issue Tracking `_", + ], + ) + def test_a_role_carrying_prose_is_not_a_no_op(self, msgid: str) -> None: + """Stripping the span takes the words a reader sees away with the target + it was protecting. 130 entries in the corpus are this shape, and until + the rule learned to look inside the backticks not one of them had ever + been queued or checked.""" + assert not classify.is_noop(msgid) + assert classify.classify(msgid).translatable + + @pytest.mark.parametrize( + "msgid", + [ + ":c:member:`base_exec_prefix `", + ":c:member:`argv `", + ":ref:`pymalloc `", + ], + ) + def test_an_identifier_in_the_display_text_is_still_a_no_op(self, msgid: str) -> None: + """Counted between the spaces rather than by running the word pattern + over the text, which would find ``base``, ``exec`` and ``prefix`` in one + identifier and call that a sentence. 136 entries turn on the difference + and every one of them is a struct field.""" + assert classify.is_noop(msgid) + + def test_a_role_with_no_display_text_is_untouched(self) -> None: + assert classify.is_noop(":ref:`class-attrs-and-methods`") + def test_a_single_letter_is_not_a_word(self) -> None: """One letter beside markup is a label, not a sentence.""" assert classify.is_noop(":class:`x` a") diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 7f055d4..c2010de 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -57,14 +57,18 @@ #: left prose, which is a cheap direction: they are code, they were being sent #: to a model, and the model was returning them unchanged and being refused for #: it. One batch fewer on a full run. +#: And again for the no-op rule learning to read the prose inside a role. 130 +#: entries leave ``noop`` for ``prose``, which is the direction that costs +#: money: they have never been queued and never been checked, and both of those +#: were silent. +2 batches. EXPECTED_KINDS = { - "prose": 75_593, - "noop": 5_721, + "prose": 75_723, + "noop": 5_591, "doctest": 2_121, "literal_block": 2_923, "version_marker": 650, } -EXPECTED_BATCHES = 2_801 +EXPECTED_BATCHES = 2_803 EXPECTED_CHANGELOG_BATCHES = 576 EXPECTED_OVERSIZED = 6 EXPECTED_LARGEST_ENTRY = 12_707 diff --git a/tests/test_sync.py b/tests/test_sync.py index b8b087b..561faba 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -74,10 +74,14 @@ def test_human_segments_skip_code_that_was_copied_correctly_too() -> None: def test_human_segments_keep_a_no_op_a_person_translated() -> None: - """Only code is dropped, not everything the classifier calls non-prose. One - no-op in the corpus is a ``:ref:`` whose display text a person translated - correctly, and that is a bug in ``is_noop`` rather than a licence to throw - the translation away.""" + """Only code is dropped, not everything the classifier calls non-prose. + + This entry is the one that found the ``is_noop`` bug, back when it was a + no-op: a ``:ref:`` whose display text a person had translated correctly, + which made it the one non-prose entry in the mirror whose translation was + not a copy of its source. The classifier calls it prose now and the + assertion is unchanged, because the point was never what kind it is. A + translation is kept unless the entry is code.""" noop = reviewed( ":ref:`Documentation on attributes `.", ":ref:`Tài liệu về các thuộc tính `.", diff --git a/uv.lock b/uv.lock index bca7aab..1caf606 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.9" +version = "0.1.10" source = { editable = "." } dependencies = [ { name = "httpx" },