From 8c442e7b55f489a96ce9fba73aa2ec2ef4fc9ca7 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:41:03 +0700 Subject: [PATCH] Make an identifier need a dot, an underscore or a digit is_version_marker matched any single token of identifier characters, and a single English word is a single token of identifier characters. So Footnotes was an identifier. So were Availability, Examples, Meaning, Exceptions, Author and 1335 other ordinary words: 9366 entries over the corpus, every one of them a heading or a table cell that wants translating. The first paragraph of classify.py says a false positive here leaves an English sentence sitting in the corpus wearing a translation's clothes. That is not a hypothetical any more and this rule is what produced it. 2808 of those entries were copied through from the msgid and stamped passthrough=version_marker, which is a claim that the string needs no translation: Availability 62 times, Exceptions 25, Author 21, Introduction 18, Description 15, Notes 11. English headings written into a Vietnamese catalog and certified by the tool that wrote them. The other 6558 already had a person's translation and were hidden a different way. A non-translatable kind is excluded from Corpus.prose(), so no check that reads a translation was looking at any of them. 41 of the words are rendered more than one way across 988 entries, and the disagreements are not stylistic: sys is he thong in 38 and sys in 20, os is he dieu hanh in all 28, object is su vat in 50 and vat the in 23 where both mean a physical thing, string is soi day in 8 which is a length of rope, statement is tuyen bo in 19 which is a public declaration. The protection classify.py's first paragraph describes was never this function's doing. :mod:`asyncio` and ``sys`` reach is_noop, because stripping the markup leaves nothing behind, so narrowing the rule here does not weaken it. This function only ever sees a word with no markup on it, and for those the safe direction is the one the module already states. An identifier now needs a dot, an underscore or a digit in it, which keeps os.path, __init__, size_t, PyMem_RawMalloc and 3.14. The dotted form is written segment by segment rather than as one character class so that a trailing dot does not count, because Success. is a one-word sentence. Not free: the 2808 go back to being untranslated, which is what they are, and a full run grows by 43 batches. --- src/pydocvi/classify.py | 63 +++++++++++++++++++++++++++++++++++++++-- tests/test_apply.py | 6 ++-- tests/test_classify.py | 34 ++++++++++++++++++++-- tests/test_corpus.py | 26 ++++++++++------- uv.lock | 2 +- 5 files changed, 112 insertions(+), 19 deletions(-) diff --git a/src/pydocvi/classify.py b/src/pydocvi/classify.py index 3de0344..c63bfb2 100644 --- a/src/pydocvi/classify.py +++ b/src/pydocvi/classify.py @@ -28,7 +28,17 @@ #: A version number, a dotted identifier, a bare token: whatever a person would #: write the same way in any language. _VERSION = re.compile(r"^\d+(\.\d+)*$") -_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*$") + +#: A dotted identifier, each segment of which is an identifier. Written segment +#: by segment rather than as one character class so that a trailing dot does not +#: count: ``Success.`` is a one-word sentence and ``os.path`` is a module. +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$") + +#: What makes an identifier-shaped string an identifier rather than a word. +#: +#: A dot, an underscore or a digit. Without one of the three, ``Footnotes`` and +#: ``sys`` are the same shape and this module has no way to tell them apart. +_QUALIFIED = re.compile(r"[._0-9]") _DOCTEST = ">>>" _CONTINUED = "..." @@ -217,11 +227,58 @@ def _tight(line: str) -> bool: def is_version_marker(msgid: str) -> bool: - """Whether the entry is a version number or a bare identifier.""" + """Whether the entry is a version number or a qualified identifier. + + Qualified is the load-bearing word and it was not here at first. The rule + used to be that any single token of identifier characters was an identifier, + and a single English word is a single token of identifier characters. So + ``Footnotes`` was an identifier. So were ``Availability``, ``Examples``, + ``Meaning``, ``Exceptions``, ``Author`` and 1 335 other ordinary words: + 9 366 entries over the corpus, every one of them a heading or a table cell + that wants translating. + + The first paragraph of this module says a false positive here leaves an + English sentence sitting in the corpus wearing a translation's clothes. That + is not a hypothetical any more and this rule is what produced it. 2 808 of + those entries were copied through from the ``msgid`` and stamped + ``passthrough=version_marker``, which is a claim that the string needs no + translation: ``Availability`` 62 times, ``Exceptions`` 25, ``Author`` 21, + ``Introduction`` 18, ``Description`` 15, ``Notes`` 11. English headings, + written into a Vietnamese catalog, certified by the tool that wrote them. + + The other 6 558 already had a person's translation, and those were hidden a + different way. A non-translatable kind is excluded from + :meth:`Corpus.prose`, so no check that reads a translation was looking at + any of them. 41 of the words are rendered more than one way across 988 + entries, and the disagreements are not stylistic: + + - ``sys`` is ``hệ thống`` in 38 entries and ``sys`` in 20. ``os`` is ``hệ + điều hành`` in all 28. Module names translated into Vietnamese, which is + the failure named at the top of this module. + - ``object`` is ``sự vật`` in 50 and ``vật thể`` in 23, both of which are a + physical thing rather than the computing sense. + - ``string`` is ``sợi dây`` in 8, which is a length of rope, and + ``statement`` is ``tuyên bố`` in 19, which is a public declaration. + + The protection the top of this module talks about is not this function's + doing and never was. ``:mod:`asyncio``` and ``` ``sys` ``` reach + :func:`is_noop`, because stripping the markup leaves nothing behind. This + function only ever sees a word with no markup on it, and for those the safe + direction is the one stated up there: a wasted call costs a call, and an + English heading in a Vietnamese page is not noticed until a reader meets it. + + So an identifier now has to look like one. A dot, an underscore or a digit + somewhere in it, which keeps ``os.path``, ``__init__``, ``size_t``, + ``PyMem_RawMalloc`` and ``3.14``, and lets go of every bare word. It is not + free: the 2 808 go back to being untranslated, which is what they are, and + a full run grows by 43 batches. + """ stripped = msgid.strip() if not stripped or "\n" in stripped: return False - return bool(_VERSION.match(stripped) or _IDENTIFIER.match(stripped)) + if _VERSION.match(stripped): + return True + return bool(_IDENTIFIER.match(stripped)) and bool(_QUALIFIED.search(stripped)) def is_noop(msgid: str) -> bool: diff --git a/tests/test_apply.py b/tests/test_apply.py index 680e787..c857be5 100644 --- a/tests/test_apply.py +++ b/tests/test_apply.py @@ -103,11 +103,11 @@ def test_a_passthrough_says_what_kind_it_is_and_names_no_model(self) -> None: would be a lie in the one place the audit trusts. The kind is the one classify gives, which is how a reader of the file can check the claim against the same rule the pipeline used.""" - cat = upstream(block("abs")) + cat = upstream(block("os.path")) memory = Memory( - [Segment.from_entry(Entry(msgid="abs", msgstr="abs"), source="passthrough")] + [Segment.from_entry(Entry(msgid="os.path", msgstr="os.path"), source="passthrough")] ) - entry = entry_of(applied(cat, memory), "abs") + entry = entry_of(applied(cat, memory), "os.path") assert entry.comments[-1] == "# pydocvi: passthrough=version_marker" def test_a_human_translation_from_upstream_lands_unfuzzy_and_unstamped(self) -> None: diff --git a/tests/test_classify.py b/tests/test_classify.py index 7afd6e2..4be2d87 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -150,14 +150,44 @@ def test_a_prompt_wearing_a_virtualenv_name(self) -> None: class TestVersionMarker: - @pytest.mark.parametrize("msgid", ["3.14", "3", "3.14.0", "asyncio", "os.path", "_thread"]) - def test_bare_versions_and_identifiers(self, msgid: str) -> None: + @pytest.mark.parametrize( + "msgid", + ["3.14", "3", "3.14.0", "os.path", "_thread", "__init__", "size_t", "PyMem_RawMalloc"], + ) + def test_bare_versions_and_qualified_identifiers(self, msgid: str) -> None: assert classify.is_version_marker(msgid) @pytest.mark.parametrize("msgid", ["version 3.14", "3.14 and later", "", "a b"]) def test_anything_with_a_second_token(self, msgid: str) -> None: assert not classify.is_version_marker(msgid) + @pytest.mark.parametrize( + "msgid", ["Footnotes", "Availability", "Examples", "Meaning", "sys", "os", "asyncio"] + ) + def test_a_bare_word_is_not_an_identifier(self, msgid: str) -> None: + """These are headings and table cells, and the looser rule that called + them identifiers reached 9 366 entries. 2 808 of those it copied through + as English and stamped as needing no translation, and the other 6 558 it + took out of everything that checks what a translation says, which is how + ``sys`` came to be ``hệ thống`` in 38 entries and ``os`` to be ``hệ điều + hành`` in all 28.""" + assert not classify.is_version_marker(msgid) + assert classify.classify(msgid).translatable + + def test_a_one_word_sentence_is_not_a_dotted_name(self) -> None: + """A trailing dot is punctuation. Writing the rule as one character + class rather than segment by segment lets ``Success.`` through it.""" + assert not classify.is_version_marker("Success.") + + @pytest.mark.parametrize("msgid", [":mod:`asyncio`", "``sys``", ":class:`frozenset`"]) + def test_the_markup_this_module_worries_about_is_a_no_op_not_a_marker(self, msgid: str) -> None: + """The reason narrowing the rule above is safe. Getting ``:mod:`asyncio``` + back as ``:mod:`không đồng bộ``` is what the module docstring calls the + embarrassing failure, and nothing in the identifier rule was ever + standing between the corpus and it: strip the markup and there is no word + left, so these reach :func:`is_noop` and stop there.""" + assert classify.classify(msgid) is Kind.NOOP + class TestClassify: def test_prose_is_the_only_translatable_kind(self) -> None: diff --git a/tests/test_corpus.py b/tests/test_corpus.py index d2ef8f3..3e02e72 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -26,20 +26,26 @@ EXPECTED_CHARACTERS = 12_526_506 EXPECTED_HUMAN = 1_435 -#: Re-measured after the code rules landed. 917 entries moved out of prose and -#: none moved into it, which is what these numbers are here to make visible: the -#: classifier decides what a run costs and what it never looks at, and both are -#: silent from any other angle. The 20 that left ``noop`` and the 4 that left -#: ``version_marker`` were already passthrough and only changed which rule -#: explains them. +#: Re-measured after the identifier rule was narrowed to need a dot, an +#: underscore or a digit. 3 193 entries left ``version_marker``: 3 164 to prose +#: and 29 to ``noop``, the latter being single letters that no rule but the +#: word rule ever explained. +#: +#: This is the first re-measure where the movement is towards prose, and that +#: direction is the expensive one. It is +43 batches on a full run and 2 808 +#: entries in the content repo that go back to being untranslated, because that +#: is what they were: English headings the old rule copied through and stamped +#: as needing no translation. The classifier decides what a run costs and what +#: it never looks at, and both are silent from any other angle, which is why +#: these numbers are written down. EXPECTED_KINDS = { - "prose": 72_496, - "noop": 5_692, + "prose": 75_660, + "noop": 5_721, "doctest": 2_121, "literal_block": 2_856, - "version_marker": 3_843, + "version_marker": 650, } -EXPECTED_BATCHES = 2_759 +EXPECTED_BATCHES = 2_802 EXPECTED_CHANGELOG_BATCHES = 576 EXPECTED_OVERSIZED = 6 EXPECTED_LARGEST_ENTRY = 12_707 diff --git a/uv.lock b/uv.lock index 22fdb74..7e5b028 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.2" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "httpx" },