From 566e56658a65bdfedc312a5ee5eb922655a4b70a Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:20:03 +0700 Subject: [PATCH] Let the glossary decide which single words L02 leaves alone L02 asks whether a translation is the English handed back, and for an entry reading `sys` the answer is yes and it is the right answer. Narrowing the identifier rule put 6 558 single-word entries into prose for the first time and took this check from 10 findings to 144, a third of which were index entries naming a module or a statement that a reviewer correctly left in English. Nothing in the string separates those from `module`, `object` and `type`, which are the other side of the same list, ordinary English words used as index categories and sitting untranslated. `sys` and `Notes` are the same shape. So the decision is made once in the glossary, where `keep_en` already means it, and L02 skips an entry whose whole msgid is a kept term. G03 then checks that term in both directions. Matched on the whole msgid rather than on a substring. A kept term inside a sentence says nothing about whether the sentence was translated. On the corpus: L02 139 to 85, G03 81 to 95, G02 191 to 190, G04 143 to 142. 54 fewer rather than the 38 the issue predicted. The glossary already held `type` and `Boolean` as kept and the issue's triage had counted those 16 as real findings. The glossary wins because it is the written decision, and the disagreement is now visible rather than being an argument nobody was having. --- pyproject.toml | 2 +- src/pydocvi/audit/language.py | 23 +++++++++++++++++++++++ tests/test_audit_language.py | 35 +++++++++++++++++++++++++++++++++++ uv.lock | 2 +- 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fceea1e..a7901e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.8" +version = "0.1.9" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/audit/language.py b/src/pydocvi/audit/language.py index 37ac6f0..a8ef928 100644 --- a/src/pydocvi/audit/language.py +++ b/src/pydocvi/audit/language.py @@ -154,10 +154,33 @@ def l02_not_the_source(corpus: Corpus) -> Iterator[Finding]: what they are for. Everything else that is identical is either a model refusing the work or a classifier that should have called it passthrough, and both are worth a line in the report. + + An entry whose whole ``msgid`` is a term the glossary keeps in English is + exempt too, and that exemption is the reason the glossary has a ``keep_en`` + field at all. Narrowing the identifier rule put 6 558 single-word entries + into :meth:`Corpus.prose` for the first time and took this check from 10 + findings to 144. A third of them were entries reading ``sys``, ``builtins``, + ``import``, ``exec``, ``NaN`` and ``Infinity``, all of which are index + entries naming a module or a statement, and all of which a reviewer left in + English because that is what a Vietnamese programmer calls them. + + Nothing in the string can tell those from ``module``, ``object`` and + ``type``, which are the other 89 and are ordinary English words used as + index categories. ``sys`` and ``Notes`` are the same shape, and that is the + discrimination the classifier was narrowed for being unable to make. So it + is made once, by hand, in the glossary, where it is a written decision that + ``G03`` then checks in both directions rather than an exception buried here. + + Matched on the whole ``msgid`` and not on a substring. A kept term inside a + sentence says nothing about whether the sentence was translated, and this + check is about the entry. """ + kept = {term.en for term in corpus.glossary.kept} if corpus.glossary is not None else set() for one, entry in corpus.translated(): if classify.classify(entry.msgid) in PASSTHROUGH: continue + if entry.msgid.strip() in kept: + continue if entry.msgstr.strip() == entry.msgid.strip(): yield Finding( check="L02", diff --git a/tests/test_audit_language.py b/tests/test_audit_language.py index 52100d2..4f7ffae 100644 --- a/tests/test_audit_language.py +++ b/tests/test_audit_language.py @@ -5,8 +5,12 @@ which correct translations those are. """ +from dataclasses import replace + from conftest import catalog_of, corpus_of, entry, findings, machine_segment from pydocvi.audit import language +from pydocvi.audit.model import Corpus +from pydocvi.glossary import Glossary, Term from pydocvi.memory import Memory @@ -14,6 +18,12 @@ def over(msgid: str, msgstr: str, **overrides: object) -> object: return corpus_of(catalog_of(entry(msgid, msgstr, **overrides))) +def with_kept(corpus: object, *english: str) -> Corpus: + """The same corpus with a glossary that keeps these terms in English.""" + terms = tuple(Term(en=one, vi=one, keep_en=True) for one in english) + return replace(corpus, glossary=Glossary(version=1, terms=terms)) # type: ignore[type-var] + + class TestL01: LONG = "A sentence long enough that a reader would expect a diacritic in it." @@ -51,6 +61,31 @@ def test_a_real_translation_is_clean(self) -> None: == [] ) + def test_an_entry_that_is_a_kept_term_is_the_glossary_s_business(self) -> None: + """``sys`` is an index entry naming a module and a reviewer left it in + English because that is what a Vietnamese programmer calls it. Nothing + in the string separates it from ``Notes``, so the decision is made once + in the glossary and ``G03`` checks it in both directions.""" + corpus = over("sys", "sys", flags=()) + assert len(findings(language.l02_not_the_source, corpus)) == 1 + assert findings(language.l02_not_the_source, with_kept(corpus, "sys")) == [] + + def test_a_kept_term_inside_a_sentence_exempts_nothing(self) -> None: + """Matched on the whole ``msgid``. A kept term in a sentence says + nothing about whether the sentence was translated.""" + english = "Import the sys module first." + corpus = with_kept(over(english, english), "sys") + assert len(findings(language.l02_not_the_source, corpus)) == 1 + + def test_a_word_the_glossary_does_not_keep_is_still_reported(self) -> None: + """``module`` is the other side of the same 144 findings, an ordinary + English word used as an index category, and it wants translating.""" + corpus = with_kept(over("module", "module", flags=()), "sys") + assert len(findings(language.l02_not_the_source, corpus)) == 1 + + def test_no_glossary_means_no_exemption_rather_than_a_crash(self) -> None: + assert len(findings(language.l02_not_the_source, over("sys", "sys", flags=()))) == 1 + class TestL03: def test_a_model_talking_about_the_work_is_found(self) -> None: diff --git a/uv.lock b/uv.lock index b9177c3..bca7aab 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.8" +version = "0.1.9" source = { editable = "." } dependencies = [ { name = "httpx" },