diff --git a/pyproject.toml b/pyproject.toml index a7901e0..e535559 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.9" +version = "0.1.10" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/classify.py b/src/pydocvi/classify.py index b13591c..09532ab 100644 --- a/src/pydocvi/classify.py +++ b/src/pydocvi/classify.py @@ -89,6 +89,24 @@ #: A quoted string, whose insides are data and not prose. _QUOTED = re.compile(r"'[^']*'|\"[^\"]*\"") +#: The visible half of a role or a link written with an explicit target, which +#: is ``:ref:`text ``` and ``` `text `_ ```. One pattern for both +#: because the shape is the same and so is the question about it. ``P03`` and +#: ``P06`` read the other half, the part that must not be translated. +_DISPLAY = re.compile(r"`([^`<>]+?)\s*<[^>]*>`") + +#: A whole word, anchored, so that a token has to be a word rather than contain +#: one. ``base_exec_prefix`` is one identifier and not three words. +_WHOLE = re.compile(r"^[A-Za-z]{2,}$") + +#: Punctuation to take off a token before asking whether it is a word. +_TRIM = ".,;:()[]'\"" + +#: How many words of display text stop an entry being a no-op. One word is +#: usually an identifier, and see :func:`_displayed` for why erring towards +#: leaving those alone is the cautious direction. +_PROSE_WORDS = 2 + #: A terminal prompt, with the virtualenv name some transcripts put in front of #: it. One of these anywhere in an entry makes the whole entry a transcript, #: because the lines around a prompt are the output of the command in it. @@ -369,8 +387,47 @@ def is_noop(msgid: str) -> bool: or more ASCII letters remains. ``:mod:`os.path``` is a no-op. ``the :mod:`os.path` module`` is not, and the difference is the four English words a reader would otherwise meet in the middle of a Vietnamese page. + + Unless the markup itself carries prose, which is the case this rule missed + for as long as it existed. A role written ``:ref:`text ``` has words + a reader sees inside the backticks, and stripping the span takes them away + with the target. ``:ref:`Documentation on attributes and methods on classes + ``` came out a no-op, which meant :func:`batch` + never queued it and :meth:`Corpus.prose` never showed it to a check. Seven + English words, invisible from every angle that would have reported them. + + It only surfaced because one person had translated one of these by hand, + correctly, and that made it the one non-prose entry in the mirror whose + translation was not a copy of its source. 130 entries in the corpus are this + shape. + + See :func:`_displayed` for where the line is drawn, which is the same line + :func:`is_version_marker` draws and for the same reason. + """ + return not _WORD.search(strip_markup(msgid)) and not _displayed(msgid) + + +def _displayed(msgid: str) -> bool: + """Whether a role or link in the entry carries prose a reader sees. + + Two or more whole words, counted between the spaces rather than by running + :data:`_WORD` over the text. The difference is 136 entries and every one of + them is wrong: ``:c:member:`base_exec_prefix ``` + is one identifier, and a pattern looking for runs of letters finds ``base``, + ``exec`` and ``prefix`` in it and calls that a sentence. + + So a display text of one word stays a no-op, 81 entries of ``:c:member:`` + naming a struct field. That is the same call :func:`is_version_marker` makes + about a bare token and it is deliberately the cautious one here: a missed + entry stays as it is today, and a false positive sends an identifier to a + model to be translated into something that links nowhere. """ - return not _WORD.search(strip_markup(msgid)) + return any(_words(text) >= _PROSE_WORDS for text in _DISPLAY.findall(msgid)) + + +def _words(text: str) -> int: + """How many whole words the display text has, ignoring punctuation.""" + return sum(1 for token in text.split() if _WHOLE.match(token.strip(_TRIM))) def counts(msgids: list[str]) -> Counts: diff --git a/tests/test_classify.py b/tests/test_classify.py index 8c506f9..aab54bf 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -30,6 +30,42 @@ def test_entries_with_no_prose_left(self, msgid: str) -> None: def test_entries_with_prose_around_the_markup(self, msgid: str) -> None: assert not classify.is_noop(msgid) + @pytest.mark.parametrize( + "msgid", + [ + ":ref:`Documentation on attributes and methods on classes `.", + ":ref:`A logging cookbook `", + ":doc:`Graphical User Interfaces with Tk `", + ":term:`Filesystem encoding `", + "`Issue Tracking `_", + ], + ) + def test_a_role_carrying_prose_is_not_a_no_op(self, msgid: str) -> None: + """Stripping the span takes the words a reader sees away with the target + it was protecting. 130 entries in the corpus are this shape, and until + the rule learned to look inside the backticks not one of them had ever + been queued or checked.""" + assert not classify.is_noop(msgid) + assert classify.classify(msgid).translatable + + @pytest.mark.parametrize( + "msgid", + [ + ":c:member:`base_exec_prefix `", + ":c:member:`argv `", + ":ref:`pymalloc `", + ], + ) + def test_an_identifier_in_the_display_text_is_still_a_no_op(self, msgid: str) -> None: + """Counted between the spaces rather than by running the word pattern + over the text, which would find ``base``, ``exec`` and ``prefix`` in one + identifier and call that a sentence. 136 entries turn on the difference + and every one of them is a struct field.""" + assert classify.is_noop(msgid) + + def test_a_role_with_no_display_text_is_untouched(self) -> None: + assert classify.is_noop(":ref:`class-attrs-and-methods`") + def test_a_single_letter_is_not_a_word(self) -> None: """One letter beside markup is a label, not a sentence.""" assert classify.is_noop(":class:`x` a") diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 7f055d4..c2010de 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -57,14 +57,18 @@ #: left prose, which is a cheap direction: they are code, they were being sent #: to a model, and the model was returning them unchanged and being refused for #: it. One batch fewer on a full run. +#: And again for the no-op rule learning to read the prose inside a role. 130 +#: entries leave ``noop`` for ``prose``, which is the direction that costs +#: money: they have never been queued and never been checked, and both of those +#: were silent. +2 batches. EXPECTED_KINDS = { - "prose": 75_593, - "noop": 5_721, + "prose": 75_723, + "noop": 5_591, "doctest": 2_121, "literal_block": 2_923, "version_marker": 650, } -EXPECTED_BATCHES = 2_801 +EXPECTED_BATCHES = 2_803 EXPECTED_CHANGELOG_BATCHES = 576 EXPECTED_OVERSIZED = 6 EXPECTED_LARGEST_ENTRY = 12_707 diff --git a/tests/test_sync.py b/tests/test_sync.py index b8b087b..561faba 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -74,10 +74,14 @@ def test_human_segments_skip_code_that_was_copied_correctly_too() -> None: def test_human_segments_keep_a_no_op_a_person_translated() -> None: - """Only code is dropped, not everything the classifier calls non-prose. One - no-op in the corpus is a ``:ref:`` whose display text a person translated - correctly, and that is a bug in ``is_noop`` rather than a licence to throw - the translation away.""" + """Only code is dropped, not everything the classifier calls non-prose. + + This entry is the one that found the ``is_noop`` bug, back when it was a + no-op: a ``:ref:`` whose display text a person had translated correctly, + which made it the one non-prose entry in the mirror whose translation was + not a copy of its source. The classifier calls it prose now and the + assertion is unchanged, because the point was never what kind it is. A + translation is kept unless the entry is code.""" noop = reviewed( ":ref:`Documentation on attributes `.", ":ref:`Tài liệu về các thuộc tính `.", diff --git a/uv.lock b/uv.lock index bca7aab..1caf606 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.9" +version = "0.1.10" source = { editable = "." } dependencies = [ { name = "httpx" },