Stop calling a role with prose in it a no-op

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "python-docs-vi-translator"
-version = "0.1.9"
+version = "0.1.10"
 description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes"
 readme = "README.md"
 license = "MIT"

src/pydocvi/classify.py

@@ -89,6 +89,24 @@
 #: A quoted string, whose insides are data and not prose.
 _QUOTED = re.compile(r"'[^']*'|\"[^\"]*\"")
 
+#: The visible half of a role or a link written with an explicit target, which
+#: is ``:ref:`text <target>``` and ``` `text <url>`_ ```. One pattern for both
+#: because the shape is the same and so is the question about it. ``P03`` and
+#: ``P06`` read the other half, the part that must not be translated.
+_DISPLAY = re.compile(r"`([^`<>]+?)\s*<[^>]*>`")
+
+#: A whole word, anchored, so that a token has to be a word rather than contain
+#: one. ``base_exec_prefix`` is one identifier and not three words.
+_WHOLE = re.compile(r"^[A-Za-z]{2,}$")
+
+#: Punctuation to take off a token before asking whether it is a word.
+_TRIM = ".,;:()[]'\""
+
+#: How many words of display text stop an entry being a no-op. One word is
+#: usually an identifier, and see :func:`_displayed` for why erring towards
+#: leaving those alone is the cautious direction.
+_PROSE_WORDS = 2
+
 #: A terminal prompt, with the virtualenv name some transcripts put in front of
 #: it. One of these anywhere in an entry makes the whole entry a transcript,
 #: because the lines around a prompt are the output of the command in it.
@@ -369,8 +387,47 @@ def is_noop(msgid: str) -> bool:
     or more ASCII letters remains. ``:mod:`os.path``` is a no-op. ``the
     :mod:`os.path` module`` is not, and the difference is the four English words
     a reader would otherwise meet in the middle of a Vietnamese page.
+
+    Unless the markup itself carries prose, which is the case this rule missed
+    for as long as it existed. A role written ``:ref:`text <target>``` has words
+    a reader sees inside the backticks, and stripping the span takes them away
+    with the target. ``:ref:`Documentation on attributes and methods on classes
+    <class-attrs-and-methods>``` came out a no-op, which meant :func:`batch`
+    never queued it and :meth:`Corpus.prose` never showed it to a check. Seven
+    English words, invisible from every angle that would have reported them.
+
+    It only surfaced because one person had translated one of these by hand,
+    correctly, and that made it the one non-prose entry in the mirror whose
+    translation was not a copy of its source. 130 entries in the corpus are this
+    shape.
+
+    See :func:`_displayed` for where the line is drawn, which is the same line
+    :func:`is_version_marker` draws and for the same reason.
+    """
+    return not _WORD.search(strip_markup(msgid)) and not _displayed(msgid)
+
+
+def _displayed(msgid: str) -> bool:
+    """Whether a role or link in the entry carries prose a reader sees.
+
+    Two or more whole words, counted between the spaces rather than by running
+    :data:`_WORD` over the text. The difference is 136 entries and every one of
+    them is wrong: ``:c:member:`base_exec_prefix <PyConfig.base_exec_prefix>```
+    is one identifier, and a pattern looking for runs of letters finds ``base``,
+    ``exec`` and ``prefix`` in it and calls that a sentence.
+
+    So a display text of one word stays a no-op, 81 entries of ``:c:member:``
+    naming a struct field. That is the same call :func:`is_version_marker` makes
+    about a bare token and it is deliberately the cautious one here: a missed
+    entry stays as it is today, and a false positive sends an identifier to a
+    model to be translated into something that links nowhere.
     """
-    return not _WORD.search(strip_markup(msgid))
+    return any(_words(text) >= _PROSE_WORDS for text in _DISPLAY.findall(msgid))
+
+
+def _words(text: str) -> int:
+    """How many whole words the display text has, ignoring punctuation."""
+    return sum(1 for token in text.split() if _WHOLE.match(token.strip(_TRIM)))
 
 
 def counts(msgids: list[str]) -> Counts:

tests/test_classify.py

@@ -30,6 +30,42 @@ def test_entries_with_no_prose_left(self, msgid: str) -> None:
     def test_entries_with_prose_around_the_markup(self, msgid: str) -> None:
         assert not classify.is_noop(msgid)
 
+    @pytest.mark.parametrize(
+        "msgid",
+        [
+            ":ref:`Documentation on attributes and methods on classes <class-attrs-and-methods>`.",
+            ":ref:`A logging cookbook <logging-cookbook>`",
+            ":doc:`Graphical User Interfaces with Tk <library/tk>`",
+            ":term:`Filesystem encoding <filesystem encoding and error handler>`",
+            "`Issue Tracking <https://devguide.python.org/tracker/>`_",
+        ],
+    )
+    def test_a_role_carrying_prose_is_not_a_no_op(self, msgid: str) -> None:
+        """Stripping the span takes the words a reader sees away with the target
+        it was protecting. 130 entries in the corpus are this shape, and until
+        the rule learned to look inside the backticks not one of them had ever
+        been queued or checked."""
+        assert not classify.is_noop(msgid)
+        assert classify.classify(msgid).translatable
+
+    @pytest.mark.parametrize(
+        "msgid",
+        [
+            ":c:member:`base_exec_prefix <PyConfig.base_exec_prefix>`",
+            ":c:member:`argv <PyConfig.argv>`",
+            ":ref:`pymalloc <pymalloc>`",
+        ],
+    )
+    def test_an_identifier_in_the_display_text_is_still_a_no_op(self, msgid: str) -> None:
+        """Counted between the spaces rather than by running the word pattern
+        over the text, which would find ``base``, ``exec`` and ``prefix`` in one
+        identifier and call that a sentence. 136 entries turn on the difference
+        and every one of them is a struct field."""
+        assert classify.is_noop(msgid)
+
+    def test_a_role_with_no_display_text_is_untouched(self) -> None:
+        assert classify.is_noop(":ref:`class-attrs-and-methods`")
+
     def test_a_single_letter_is_not_a_word(self) -> None:
         """One letter beside markup is a label, not a sentence."""
         assert classify.is_noop(":class:`x` a")

tests/test_corpus.py

@@ -57,14 +57,18 @@
 #: left prose, which is a cheap direction: they are code, they were being sent
 #: to a model, and the model was returning them unchanged and being refused for
 #: it. One batch fewer on a full run.
+#: And again for the no-op rule learning to read the prose inside a role. 130
+#: entries leave ``noop`` for ``prose``, which is the direction that costs
+#: money: they have never been queued and never been checked, and both of those
+#: were silent. +2 batches.
 EXPECTED_KINDS = {
-    "prose": 75_593,
-    "noop": 5_721,
+    "prose": 75_723,
+    "noop": 5_591,
     "doctest": 2_121,
     "literal_block": 2_923,
     "version_marker": 650,
 }
-EXPECTED_BATCHES = 2_801
+EXPECTED_BATCHES = 2_803
 EXPECTED_CHANGELOG_BATCHES = 576
 EXPECTED_OVERSIZED = 6
 EXPECTED_LARGEST_ENTRY = 12_707

tests/test_sync.py

@@ -74,10 +74,14 @@ def test_human_segments_skip_code_that_was_copied_correctly_too() -> None:
 
 
 def test_human_segments_keep_a_no_op_a_person_translated() -> None:
-    """Only code is dropped, not everything the classifier calls non-prose. One
-    no-op in the corpus is a ``:ref:`` whose display text a person translated
-    correctly, and that is a bug in ``is_noop`` rather than a licence to throw
-    the translation away."""
+    """Only code is dropped, not everything the classifier calls non-prose.
+
+    This entry is the one that found the ``is_noop`` bug, back when it was a
+    no-op: a ``:ref:`` whose display text a person had translated correctly,
+    which made it the one non-prose entry in the mirror whose translation was
+    not a copy of its source. The classifier calls it prose now and the
+    assertion is unchanged, because the point was never what kind it is. A
+    translation is kept unless the entry is code."""
     noop = reviewed(
         ":ref:`Documentation on attributes <class-attrs>`.",
         ":ref:`Tài liệu về các thuộc tính <class-attrs>`.",