diff --git a/src/pydocvi/classify.py b/src/pydocvi/classify.py index 3de0344..c63bfb2 100644 --- a/src/pydocvi/classify.py +++ b/src/pydocvi/classify.py @@ -28,7 +28,17 @@ #: A version number, a dotted identifier, a bare token: whatever a person would #: write the same way in any language. _VERSION = re.compile(r"^\d+(\.\d+)*$") -_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]*$") + +#: A dotted identifier, each segment of which is an identifier. Written segment +#: by segment rather than as one character class so that a trailing dot does not +#: count: ``Success.`` is a one-word sentence and ``os.path`` is a module. +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$") + +#: What makes an identifier-shaped string an identifier rather than a word. +#: +#: A dot, an underscore or a digit. Without one of the three, ``Footnotes`` and +#: ``sys`` are the same shape and this module has no way to tell them apart. +_QUALIFIED = re.compile(r"[._0-9]") _DOCTEST = ">>>" _CONTINUED = "..." @@ -217,11 +227,58 @@ def _tight(line: str) -> bool: def is_version_marker(msgid: str) -> bool: - """Whether the entry is a version number or a bare identifier.""" + """Whether the entry is a version number or a qualified identifier. + + Qualified is the load-bearing word and it was not here at first. The rule + used to be that any single token of identifier characters was an identifier, + and a single English word is a single token of identifier characters. So + ``Footnotes`` was an identifier. So were ``Availability``, ``Examples``, + ``Meaning``, ``Exceptions``, ``Author`` and 1 335 other ordinary words: + 9 366 entries over the corpus, every one of them a heading or a table cell + that wants translating. + + The first paragraph of this module says a false positive here leaves an + English sentence sitting in the corpus wearing a translation's clothes. That + is not a hypothetical any more and this rule is what produced it. 2 808 of + those entries were copied through from the ``msgid`` and stamped + ``passthrough=version_marker``, which is a claim that the string needs no + translation: ``Availability`` 62 times, ``Exceptions`` 25, ``Author`` 21, + ``Introduction`` 18, ``Description`` 15, ``Notes`` 11. English headings, + written into a Vietnamese catalog, certified by the tool that wrote them. + + The other 6 558 already had a person's translation, and those were hidden a + different way. A non-translatable kind is excluded from + :meth:`Corpus.prose`, so no check that reads a translation was looking at + any of them. 41 of the words are rendered more than one way across 988 + entries, and the disagreements are not stylistic: + + - ``sys`` is ``hệ thống`` in 38 entries and ``sys`` in 20. ``os`` is ``hệ + điều hành`` in all 28. Module names translated into Vietnamese, which is + the failure named at the top of this module. + - ``object`` is ``sự vật`` in 50 and ``vật thể`` in 23, both of which are a + physical thing rather than the computing sense. + - ``string`` is ``sợi dây`` in 8, which is a length of rope, and + ``statement`` is ``tuyên bố`` in 19, which is a public declaration. + + The protection the top of this module talks about is not this function's + doing and never was. ``:mod:`asyncio``` and ``` ``sys` ``` reach + :func:`is_noop`, because stripping the markup leaves nothing behind. This + function only ever sees a word with no markup on it, and for those the safe + direction is the one stated up there: a wasted call costs a call, and an + English heading in a Vietnamese page is not noticed until a reader meets it. + + So an identifier now has to look like one. A dot, an underscore or a digit + somewhere in it, which keeps ``os.path``, ``__init__``, ``size_t``, + ``PyMem_RawMalloc`` and ``3.14``, and lets go of every bare word. It is not + free: the 2 808 go back to being untranslated, which is what they are, and + a full run grows by 43 batches. + """ stripped = msgid.strip() if not stripped or "\n" in stripped: return False - return bool(_VERSION.match(stripped) or _IDENTIFIER.match(stripped)) + if _VERSION.match(stripped): + return True + return bool(_IDENTIFIER.match(stripped)) and bool(_QUALIFIED.search(stripped)) def is_noop(msgid: str) -> bool: diff --git a/tests/test_apply.py b/tests/test_apply.py index 680e787..c857be5 100644 --- a/tests/test_apply.py +++ b/tests/test_apply.py @@ -103,11 +103,11 @@ def test_a_passthrough_says_what_kind_it_is_and_names_no_model(self) -> None: would be a lie in the one place the audit trusts. The kind is the one classify gives, which is how a reader of the file can check the claim against the same rule the pipeline used.""" - cat = upstream(block("abs")) + cat = upstream(block("os.path")) memory = Memory( - [Segment.from_entry(Entry(msgid="abs", msgstr="abs"), source="passthrough")] + [Segment.from_entry(Entry(msgid="os.path", msgstr="os.path"), source="passthrough")] ) - entry = entry_of(applied(cat, memory), "abs") + entry = entry_of(applied(cat, memory), "os.path") assert entry.comments[-1] == "# pydocvi: passthrough=version_marker" def test_a_human_translation_from_upstream_lands_unfuzzy_and_unstamped(self) -> None: diff --git a/tests/test_classify.py b/tests/test_classify.py index 7afd6e2..4be2d87 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -150,14 +150,44 @@ def test_a_prompt_wearing_a_virtualenv_name(self) -> None: class TestVersionMarker: - @pytest.mark.parametrize("msgid", ["3.14", "3", "3.14.0", "asyncio", "os.path", "_thread"]) - def test_bare_versions_and_identifiers(self, msgid: str) -> None: + @pytest.mark.parametrize( + "msgid", + ["3.14", "3", "3.14.0", "os.path", "_thread", "__init__", "size_t", "PyMem_RawMalloc"], + ) + def test_bare_versions_and_qualified_identifiers(self, msgid: str) -> None: assert classify.is_version_marker(msgid) @pytest.mark.parametrize("msgid", ["version 3.14", "3.14 and later", "", "a b"]) def test_anything_with_a_second_token(self, msgid: str) -> None: assert not classify.is_version_marker(msgid) + @pytest.mark.parametrize( + "msgid", ["Footnotes", "Availability", "Examples", "Meaning", "sys", "os", "asyncio"] + ) + def test_a_bare_word_is_not_an_identifier(self, msgid: str) -> None: + """These are headings and table cells, and the looser rule that called + them identifiers reached 9 366 entries. 2 808 of those it copied through + as English and stamped as needing no translation, and the other 6 558 it + took out of everything that checks what a translation says, which is how + ``sys`` came to be ``hệ thống`` in 38 entries and ``os`` to be ``hệ điều + hành`` in all 28.""" + assert not classify.is_version_marker(msgid) + assert classify.classify(msgid).translatable + + def test_a_one_word_sentence_is_not_a_dotted_name(self) -> None: + """A trailing dot is punctuation. Writing the rule as one character + class rather than segment by segment lets ``Success.`` through it.""" + assert not classify.is_version_marker("Success.") + + @pytest.mark.parametrize("msgid", [":mod:`asyncio`", "``sys``", ":class:`frozenset`"]) + def test_the_markup_this_module_worries_about_is_a_no_op_not_a_marker(self, msgid: str) -> None: + """The reason narrowing the rule above is safe. Getting ``:mod:`asyncio``` + back as ``:mod:`không đồng bộ``` is what the module docstring calls the + embarrassing failure, and nothing in the identifier rule was ever + standing between the corpus and it: strip the markup and there is no word + left, so these reach :func:`is_noop` and stop there.""" + assert classify.classify(msgid) is Kind.NOOP + class TestClassify: def test_prose_is_the_only_translatable_kind(self) -> None: diff --git a/tests/test_corpus.py b/tests/test_corpus.py index d2ef8f3..3e02e72 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -26,20 +26,26 @@ EXPECTED_CHARACTERS = 12_526_506 EXPECTED_HUMAN = 1_435 -#: Re-measured after the code rules landed. 917 entries moved out of prose and -#: none moved into it, which is what these numbers are here to make visible: the -#: classifier decides what a run costs and what it never looks at, and both are -#: silent from any other angle. The 20 that left ``noop`` and the 4 that left -#: ``version_marker`` were already passthrough and only changed which rule -#: explains them. +#: Re-measured after the identifier rule was narrowed to need a dot, an +#: underscore or a digit. 3 193 entries left ``version_marker``: 3 164 to prose +#: and 29 to ``noop``, the latter being single letters that no rule but the +#: word rule ever explained. +#: +#: This is the first re-measure where the movement is towards prose, and that +#: direction is the expensive one. It is +43 batches on a full run and 2 808 +#: entries in the content repo that go back to being untranslated, because that +#: is what they were: English headings the old rule copied through and stamped +#: as needing no translation. The classifier decides what a run costs and what +#: it never looks at, and both are silent from any other angle, which is why +#: these numbers are written down. EXPECTED_KINDS = { - "prose": 72_496, - "noop": 5_692, + "prose": 75_660, + "noop": 5_721, "doctest": 2_121, "literal_block": 2_856, - "version_marker": 3_843, + "version_marker": 650, } -EXPECTED_BATCHES = 2_759 +EXPECTED_BATCHES = 2_802 EXPECTED_CHANGELOG_BATCHES = 576 EXPECTED_OVERSIZED = 6 EXPECTED_LARGEST_ENTRY = 12_707 diff --git a/uv.lock b/uv.lock index 22fdb74..7e5b028 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.2" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "httpx" },