Make an identifier need a dot, an underscore or a digit

src/pydocvi/classify.py

@@ -28,7 +28,17 @@
 #: A version number, a dotted identifier, a bare token: whatever a person would
 #: write the same way in any language.
 _VERSION = re.compile(r"^\d+(\.\d+)*$")
-_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*$")
+
+#: A dotted identifier, each segment of which is an identifier. Written segment
+#: by segment rather than as one character class so that a trailing dot does not
+#: count: ``Success.`` is a one-word sentence and ``os.path`` is a module.
+_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$")
+
+#: What makes an identifier-shaped string an identifier rather than a word.
+#:
+#: A dot, an underscore or a digit. Without one of the three, ``Footnotes`` and
+#: ``sys`` are the same shape and this module has no way to tell them apart.
+_QUALIFIED = re.compile(r"[._0-9]")
 
 _DOCTEST = ">>>"
 _CONTINUED = "..."
@@ -217,11 +227,58 @@ def _tight(line: str) -> bool:
 
 
 def is_version_marker(msgid: str) -> bool:
-    """Whether the entry is a version number or a bare identifier."""
+    """Whether the entry is a version number or a qualified identifier.
+
+    Qualified is the load-bearing word and it was not here at first. The rule
+    used to be that any single token of identifier characters was an identifier,
+    and a single English word is a single token of identifier characters. So
+    ``Footnotes`` was an identifier. So were ``Availability``, ``Examples``,
+    ``Meaning``, ``Exceptions``, ``Author`` and 1 335 other ordinary words:
+    9 366 entries over the corpus, every one of them a heading or a table cell
+    that wants translating.
+
+    The first paragraph of this module says a false positive here leaves an
+    English sentence sitting in the corpus wearing a translation's clothes. That
+    is not a hypothetical any more and this rule is what produced it. 2 808 of
+    those entries were copied through from the ``msgid`` and stamped
+    ``passthrough=version_marker``, which is a claim that the string needs no
+    translation: ``Availability`` 62 times, ``Exceptions`` 25, ``Author`` 21,
+    ``Introduction`` 18, ``Description`` 15, ``Notes`` 11. English headings,
+    written into a Vietnamese catalog, certified by the tool that wrote them.
+
+    The other 6 558 already had a person's translation, and those were hidden a
+    different way. A non-translatable kind is excluded from
+    :meth:`Corpus.prose`, so no check that reads a translation was looking at
+    any of them. 41 of the words are rendered more than one way across 988
+    entries, and the disagreements are not stylistic:
+
+    - ``sys`` is ``hệ thống`` in 38 entries and ``sys`` in 20. ``os`` is ``hệ
+      điều hành`` in all 28. Module names translated into Vietnamese, which is
+      the failure named at the top of this module.
+    - ``object`` is ``sự vật`` in 50 and ``vật thể`` in 23, both of which are a
+      physical thing rather than the computing sense.
+    - ``string`` is ``sợi dây`` in 8, which is a length of rope, and
+      ``statement`` is ``tuyên bố`` in 19, which is a public declaration.
+
+    The protection the top of this module talks about is not this function's
+    doing and never was. ``:mod:`asyncio``` and ``` ``sys` ``` reach
+    :func:`is_noop`, because stripping the markup leaves nothing behind. This
+    function only ever sees a word with no markup on it, and for those the safe
+    direction is the one stated up there: a wasted call costs a call, and an
+    English heading in a Vietnamese page is not noticed until a reader meets it.
+
+    So an identifier now has to look like one. A dot, an underscore or a digit
+    somewhere in it, which keeps ``os.path``, ``__init__``, ``size_t``,
+    ``PyMem_RawMalloc`` and ``3.14``, and lets go of every bare word. It is not
+    free: the 2 808 go back to being untranslated, which is what they are, and
+    a full run grows by 43 batches.
+    """
     stripped = msgid.strip()
     if not stripped or "\n" in stripped:
         return False
-    return bool(_VERSION.match(stripped) or _IDENTIFIER.match(stripped))
+    if _VERSION.match(stripped):
+        return True
+    return bool(_IDENTIFIER.match(stripped)) and bool(_QUALIFIED.search(stripped))
 
 
 def is_noop(msgid: str) -> bool:

tests/test_apply.py

@@ -103,11 +103,11 @@ def test_a_passthrough_says_what_kind_it_is_and_names_no_model(self) -> None:
         would be a lie in the one place the audit trusts. The kind is the one
         classify gives, which is how a reader of the file can check the claim
         against the same rule the pipeline used."""
-        cat = upstream(block("abs"))
+        cat = upstream(block("os.path"))
         memory = Memory(
-            [Segment.from_entry(Entry(msgid="abs", msgstr="abs"), source="passthrough")]
+            [Segment.from_entry(Entry(msgid="os.path", msgstr="os.path"), source="passthrough")]
         )
-        entry = entry_of(applied(cat, memory), "abs")
+        entry = entry_of(applied(cat, memory), "os.path")
         assert entry.comments[-1] == "# pydocvi: passthrough=version_marker"
 
     def test_a_human_translation_from_upstream_lands_unfuzzy_and_unstamped(self) -> None:

tests/test_classify.py

@@ -150,14 +150,44 @@ def test_a_prompt_wearing_a_virtualenv_name(self) -> None:
 
 
 class TestVersionMarker:
-    @pytest.mark.parametrize("msgid", ["3.14", "3", "3.14.0", "asyncio", "os.path", "_thread"])
-    def test_bare_versions_and_identifiers(self, msgid: str) -> None:
+    @pytest.mark.parametrize(
+        "msgid",
+        ["3.14", "3", "3.14.0", "os.path", "_thread", "__init__", "size_t", "PyMem_RawMalloc"],
+    )
+    def test_bare_versions_and_qualified_identifiers(self, msgid: str) -> None:
         assert classify.is_version_marker(msgid)
 
     @pytest.mark.parametrize("msgid", ["version 3.14", "3.14 and later", "", "a b"])
     def test_anything_with_a_second_token(self, msgid: str) -> None:
         assert not classify.is_version_marker(msgid)
 
+    @pytest.mark.parametrize(
+        "msgid", ["Footnotes", "Availability", "Examples", "Meaning", "sys", "os", "asyncio"]
+    )
+    def test_a_bare_word_is_not_an_identifier(self, msgid: str) -> None:
+        """These are headings and table cells, and the looser rule that called
+        them identifiers reached 9 366 entries. 2 808 of those it copied through
+        as English and stamped as needing no translation, and the other 6 558 it
+        took out of everything that checks what a translation says, which is how
+        ``sys`` came to be ``hệ thống`` in 38 entries and ``os`` to be ``hệ điều
+        hành`` in all 28."""
+        assert not classify.is_version_marker(msgid)
+        assert classify.classify(msgid).translatable
+
+    def test_a_one_word_sentence_is_not_a_dotted_name(self) -> None:
+        """A trailing dot is punctuation. Writing the rule as one character
+        class rather than segment by segment lets ``Success.`` through it."""
+        assert not classify.is_version_marker("Success.")
+
+    @pytest.mark.parametrize("msgid", [":mod:`asyncio`", "``sys``", ":class:`frozenset`"])
+    def test_the_markup_this_module_worries_about_is_a_no_op_not_a_marker(self, msgid: str) -> None:
+        """The reason narrowing the rule above is safe. Getting ``:mod:`asyncio```
+        back as ``:mod:`không đồng bộ``` is what the module docstring calls the
+        embarrassing failure, and nothing in the identifier rule was ever
+        standing between the corpus and it: strip the markup and there is no word
+        left, so these reach :func:`is_noop` and stop there."""
+        assert classify.classify(msgid) is Kind.NOOP
+
 
 class TestClassify:
     def test_prose_is_the_only_translatable_kind(self) -> None:

tests/test_corpus.py

@@ -26,20 +26,26 @@
 EXPECTED_CHARACTERS = 12_526_506
 EXPECTED_HUMAN = 1_435
 
-#: Re-measured after the code rules landed. 917 entries moved out of prose and
-#: none moved into it, which is what these numbers are here to make visible: the
-#: classifier decides what a run costs and what it never looks at, and both are
-#: silent from any other angle. The 20 that left ``noop`` and the 4 that left
-#: ``version_marker`` were already passthrough and only changed which rule
-#: explains them.
+#: Re-measured after the identifier rule was narrowed to need a dot, an
+#: underscore or a digit. 3 193 entries left ``version_marker``: 3 164 to prose
+#: and 29 to ``noop``, the latter being single letters that no rule but the
+#: word rule ever explained.
+#:
+#: This is the first re-measure where the movement is towards prose, and that
+#: direction is the expensive one. It is +43 batches on a full run and 2 808
+#: entries in the content repo that go back to being untranslated, because that
+#: is what they were: English headings the old rule copied through and stamped
+#: as needing no translation. The classifier decides what a run costs and what
+#: it never looks at, and both are silent from any other angle, which is why
+#: these numbers are written down.
 EXPECTED_KINDS = {
-    "prose": 72_496,
-    "noop": 5_692,
+    "prose": 75_660,
+    "noop": 5_721,
     "doctest": 2_121,
     "literal_block": 2_856,
-    "version_marker": 3_843,
+    "version_marker": 650,
 }
-EXPECTED_BATCHES = 2_759
+EXPECTED_BATCHES = 2_802
 EXPECTED_CHANGELOG_BATCHES = 576
 EXPECTED_OVERSIZED = 6
 EXPECTED_LARGEST_ENTRY = 12_707