From b629cc4be94db1931370b46d3dce869a6c6b86a1 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:04:39 +0700 Subject: [PATCH 1/2] Let one glossary row answer both questions about a word L02 exempted an entry whose whole msgid is a term the glossary keeps in English, and read keep_en to find them. That made keep_en carry two decisions at once: how the word reads in a sentence, and whether an entry that is only the word is already right. For most rows those are the same answer. For six of them they are not. float is so thuc in a sentence and the name of a C type in the table of struct format codes. type is kieu in 44 of the 49 prose entries that translate it, and the argparse keyword argument in a table of argparse keyword arguments. list is danh sach in 38 of 55, and the Python type in the json conversion table. While the two questions shared a field, a row could only answer one, and the answer that got written was the one that silenced the louder check. 69 of G03's 94 findings are correct translations of type and list, held there by rows that say "keep this in English" when what they meant was "leave the table cell alone". So Term gains identifier, Glossary gains standalone, and L02 reads the second one. A keep_en row is standalone without being told, because a term that is English in every sentence is English on its own too, which means no existing row changes meaning and the glossary file is the only place the new decisions get made. The prompt says both things too, on one line. library/struct.po has float as a table cell two entries away from a sentence about floats, so a batch holds both readings and a line carrying only the rendering would get the cell translated. --- pyproject.toml | 2 +- src/pydocvi/audit/language.py | 40 ++++++++++++++++++++------------ src/pydocvi/glossary.py | 43 ++++++++++++++++++++++++++++++++--- src/pydocvi/render.py | 8 +++++++ tests/test_audit_language.py | 27 +++++++++++++++++++--- tests/test_glossary.py | 40 +++++++++++++++++++++++++++++++- tests/test_render.py | 14 ++++++++++++ 7 files changed, 152 insertions(+), 22 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e535559..c6537ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.10" +version = "0.1.11" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/audit/language.py b/src/pydocvi/audit/language.py index a8ef928..375b708 100644 --- a/src/pydocvi/audit/language.py +++ b/src/pydocvi/audit/language.py @@ -155,27 +155,39 @@ def l02_not_the_source(corpus: Corpus) -> Iterator[Finding]: refusing the work or a classifier that should have called it passthrough, and both are worth a line in the report. - An entry whose whole ``msgid`` is a term the glossary keeps in English is - exempt too, and that exemption is the reason the glossary has a ``keep_en`` - field at all. Narrowing the identifier rule put 6 558 single-word entries - into :meth:`Corpus.prose` for the first time and took this check from 10 - findings to 144. A third of them were entries reading ``sys``, ``builtins``, - ``import``, ``exec``, ``NaN`` and ``Infinity``, all of which are index - entries naming a module or a statement, and all of which a reviewer left in - English because that is what a Vietnamese programmer calls them. + An entry whose whole ``msgid`` is a term the glossary marks as standing + alone is exempt too, and that exemption is the reason the glossary carries + the flag. Narrowing the identifier rule put 6 558 single-word entries into + :meth:`Corpus.prose` for the first time and took this check from 10 findings + to 144. A third of them were entries reading ``sys``, ``builtins``, + ``import``, ``exec`` and ``NaN``, all of which are index entries naming a + module or a statement, and all of which a reviewer left in English because + that is what a Vietnamese programmer calls them. Nothing in the string can tell those from ``module``, ``object`` and ``type``, which are the other 89 and are ordinary English words used as index categories. ``sys`` and ``Notes`` are the same shape, and that is the discrimination the classifier was narrowed for being unable to make. So it - is made once, by hand, in the glossary, where it is a written decision that - ``G03`` then checks in both directions rather than an exception buried here. - - Matched on the whole ``msgid`` and not on a substring. A kept term inside a + is made once, by hand, in the glossary, where it is a written decision + rather than an exception buried here. + + Read from :attr:`Glossary.standalone` and not from ``keep_en``, because the + two questions came apart. ``float`` is ``số thực`` in a sentence and the + name of a C type in the table of ``struct`` format codes, and while this + check read ``keep_en`` a row could only answer one of those. 69 of the 94 + findings ``G03`` was reporting were correct translations of ``type`` and + ``list``, held there by rows that said "keep this in English" when what they + meant was "leave the table cell alone". + + Matched on the whole ``msgid`` and not on a substring. A term inside a sentence says nothing about whether the sentence was translated, and this - check is about the entry. + check is about the entry. Nor is the match folded or de-inflected: ``Lists`` + is a section heading three times in the corpus, followed each time by prose + beginning "Lists are mutable sequences", and a heading is translated. """ - kept = {term.en for term in corpus.glossary.kept} if corpus.glossary is not None else set() + kept = ( + {term.en for term in corpus.glossary.standalone} if corpus.glossary is not None else set() + ) for one, entry in corpus.translated(): if classify.classify(entry.msgid) in PASSTHROUGH: continue diff --git a/src/pydocvi/glossary.py b/src/pydocvi/glossary.py index c4e40f4..9199581 100644 --- a/src/pydocvi/glossary.py +++ b/src/pydocvi/glossary.py @@ -52,6 +52,15 @@ class Term: such, and ``G02`` then checks the opposite thing: that the English survived rather than that it was replaced. + ``identifier`` is the field for the words that are both. "float" in a + sentence is "số thực" and the corpus says so twice; "float" on its own in the + table of struct format codes is the name of a C type and translating it would + break the table. One row has to be able to say both things, because the + alternative is what the file did before: pick the reading that suits the + louder check and be wrong about the other. ``keep_en`` answers the question + for running prose, ``identifier`` answers it for an entry that is nothing but + the term, and ``L02`` is the only check that asks the second one. + ``context`` is a filter, not prose. It is a fragment matched against the entry's ``msgctxt`` and against the file path, so a row carrying one applies only where that fragment appears. Prose about a row belongs in ``note``, @@ -62,6 +71,7 @@ class Term: en: str vi: str keep_en: bool = False + identifier: bool = False context: str | None = None note: str = "" @@ -123,6 +133,16 @@ def translated(self) -> tuple[Term, ...]: def kept(self) -> tuple[Term, ...]: return tuple(term for term in self.terms if term.keep_en) + @property + def standalone(self) -> tuple[Term, ...]: + """The rows an entry may equal and still be correct in English. + + A ``keep_en`` row is one of these without saying so: a term that stays + English everywhere stays English standing alone too. The rows that need + the flag are the ones translated in a sentence and named in a table. + """ + return tuple(term for term in self.terms if term.keep_en or term.identifier) + def with_terms(self, terms: Iterable[Term], *, version: int | None = None) -> Self: rows = tuple(terms) bumped = version if version is not None else self.version + bool(_changed(self.terms, rows)) @@ -344,7 +364,12 @@ def _differs(before: Term, after: Term) -> bool: ``note`` is not in here. A note is written for the person reading the file and changing one is not a reason to re-queue a few hundred entries. """ - return (before.vi, before.keep_en, before.context) != (after.vi, after.keep_en, after.context) + return (before.vi, before.keep_en, before.identifier, before.context) != ( + after.vi, + after.keep_en, + after.identifier, + after.context, + ) def _changed(before: Sequence[Term], after: Sequence[Term]) -> bool: @@ -500,7 +525,7 @@ def loads(text: str) -> Glossary: def _row(raw: object, at: int) -> Term: if not isinstance(raw, dict): raise GlossaryError(f"term {at} is not a mapping") - unknown = set(raw) - {"en", "vi", "keep_en", "context", "note"} + unknown = set(raw) - {"en", "vi", "keep_en", "identifier", "context", "note"} if unknown: raise GlossaryError(f"term {at} has unknown field(s): {', '.join(sorted(unknown))}") try: @@ -508,6 +533,7 @@ def _row(raw: object, at: int) -> Term: en=str(raw["en"]), vi=str(raw["vi"]), keep_en=bool(raw.get("keep_en", False)), + identifier=bool(raw.get("identifier", False)), context=str(raw["context"]) if raw.get("context") else None, note=str(raw.get("note", "")), ) @@ -537,6 +563,8 @@ def dumps(glossary: Glossary) -> str: out.append(f" vi: {scalar(term.vi)}") if term.keep_en: out.append(" keep_en: true") + if term.identifier: + out.append(" identifier: true") if term.context is not None: out.append(f" context: {scalar(term.context)}") if term.note: @@ -592,11 +620,20 @@ def table(glossary: Glossary) -> str: ] for term in match_order(glossary.terms): vi = f"`{term.en}` (kept)" if term.keep_en else term.vi - note = " ".join(part for part in (_context_note(term), term.note) if part) + note = " ".join( + part for part in (_context_note(term), _standalone_note(term), term.note) if part + ) lines.append(f"| {term.en} | {vi} | {_cell(note)} |") return "\n".join(lines) +def _standalone_note(term: Term) -> str: + """Said in the table because a reviewer reading one row cannot infer it.""" + if not term.identifier or term.keep_en: + return "" + return f"An entry that is only `{term.en}` names the thing and stays English." + + def _context_note(term: Term) -> str: return f"Only where the path or msgctxt contains `{term.context}`." if term.context else "" diff --git a/src/pydocvi/render.py b/src/pydocvi/render.py index a6ef63f..0bddff2 100644 --- a/src/pydocvi/render.py +++ b/src/pydocvi/render.py @@ -38,6 +38,13 @@ #: what to do rather than show a rendering that looks like a no-op. KEEP = "leave in English" +#: What an ``identifier`` row adds to its line. Such a row is translated in a +#: sentence and left alone when the whole string is the term, and a batch is +#: quite capable of holding both: ``library/struct.po`` has "float" as a cell in +#: the format-code table two entries away from a sentence about floats. Saying +#: only the rendering would get the cell translated. +ALONE = " (a string that is only this word names the thing: leave it in English)" + #: The first line of the user message names the file and what kind of writing it #: is, because the register is not a detail. The tutorial is addressed to a #: beginner and the C API reference is addressed to somebody writing an @@ -134,6 +141,7 @@ def terminology(terms: Sequence[Term]) -> str: return NO_TERMS return "\n".join( f"- {term.en} -> {KEEP if term.keep_en else term.vi}" + + (ALONE if term.identifier and not term.keep_en else "") + (f" ({term.note})" if term.note else "") for term in terms ) diff --git a/tests/test_audit_language.py b/tests/test_audit_language.py index 4f7ffae..0b1348e 100644 --- a/tests/test_audit_language.py +++ b/tests/test_audit_language.py @@ -24,6 +24,12 @@ def with_kept(corpus: object, *english: str) -> Corpus: return replace(corpus, glossary=Glossary(version=1, terms=terms)) # type: ignore[type-var] +def with_standalone(corpus: object, en: str, vi: str) -> Corpus: + """The same corpus with one row translated in prose and kept on its own.""" + terms = (Term(en=en, vi=vi, identifier=True),) + return replace(corpus, glossary=Glossary(version=1, terms=terms)) # type: ignore[type-var] + + class TestL01: LONG = "A sentence long enough that a reader would expect a diacritic in it." @@ -78,9 +84,24 @@ def test_a_kept_term_inside_a_sentence_exempts_nothing(self) -> None: assert len(findings(language.l02_not_the_source, corpus)) == 1 def test_a_word_the_glossary_does_not_keep_is_still_reported(self) -> None: - """``module`` is the other side of the same 144 findings, an ordinary - English word used as an index category, and it wants translating.""" - corpus = with_kept(over("module", "module", flags=()), "sys") + """``object`` is the other side of the same 144 findings, an ordinary + English word used as an index category, and it wants translating. It is + one of 36 such entries and the 124 sentences around them say ``đối + tượng`` 93 times.""" + corpus = with_kept(over("object", "object", flags=()), "sys") + assert len(findings(language.l02_not_the_source, corpus)) == 1 + + def test_a_row_translated_in_prose_can_still_stand_alone(self) -> None: + """``float`` is ``số thực`` in a sentence and the name of a C type in + the ``struct`` format table. While this check read ``keep_en`` the row + could say one or the other, and saying "keep the English" to get the + table cell right made ``G03`` report both correct translations of it.""" + corpus = with_standalone(over("float", "float", flags=()), "float", "số thực") + assert findings(language.l02_not_the_source, corpus) == [] + + def test_standing_alone_does_not_excuse_a_sentence(self) -> None: + english = "Divide and get a float." + corpus = with_standalone(over(english, english), "float", "số thực") assert len(findings(language.l02_not_the_source, corpus)) == 1 def test_no_glossary_means_no_exemption_rather_than_a_crash(self) -> None: diff --git a/tests/test_glossary.py b/tests/test_glossary.py index eddc4d9..8a634e9 100644 --- a/tests/test_glossary.py +++ b/tests/test_glossary.py @@ -75,6 +75,20 @@ def test_kept_and_translated_rows_are_separable(self): assert [row.en for row in rows.kept] == ["decorator"] assert [row.en for row in rows.translated] == ["list"] + def test_a_kept_row_stands_alone_without_being_told_to(self): + """A term that is English in every sentence is English on its own too, + so the flag is only ever needed by rows that carry a rendering.""" + rows = make(term("decorator", "decorator", keep_en=True), term("list", "danh sách")) + assert [row.en for row in rows.standalone] == ["decorator"] + + def test_a_translated_row_can_still_stand_alone(self): + """The case the field was added for. ``float`` is ``số thực`` in a + sentence and the name of a C type in the ``struct`` format table.""" + rows = make(term("float", "số thực", identifier=True), term("list", "danh sách")) + assert [row.en for row in rows.standalone] == ["float"] + assert [row.en for row in rows.translated] == ["float", "list"] + assert rows.kept == () + def test_len_counts_rows(self): assert len(make(*NESTED)) == 2 @@ -287,6 +301,13 @@ def test_flipping_keep_en_is_a_change(self): after = make(term("decorator", "decorator", keep_en=True), version=2) assert len(glossary.diff(before, after).changed) == 1 + def test_flipping_identifier_is_a_change_too(self): + """It moves entries in and out of ``L02`` and changes what the prompt + says, so a run made before it is not a run made after it.""" + before = make(term("float", "số thực")) + after = make(term("float", "số thực", identifier=True), version=2) + assert len(glossary.diff(before, after).changed) == 1 + def test_adding_a_context_is_a_change(self): before = make(term("list", "danh sách")) after = make(term("list", "danh sách", context="stdtypes"), version=2) @@ -451,10 +472,12 @@ def test_every_field_survives(self): text = ( "version: 1\nterms:\n" ' - en: "decorator"\n vi: "decorator"\n keep_en: true\n' + " identifier: true\n" ' context: "library"\n note: "the community keeps the English"\n' ) row = glossary.loads(text).terms[0] - assert (row.keep_en, row.context, row.note) == ( + assert (row.keep_en, row.identifier, row.context, row.note) == ( + True, True, "library", "the community keeps the English", @@ -513,6 +536,10 @@ def test_keep_en_is_written_only_when_it_is_true(self): assert "keep_en" not in glossary.dumps(make(term("list", "danh sách"))) assert "keep_en: true" in glossary.dumps(make(term("decorator", "decorator", keep_en=True))) + def test_identifier_is_written_only_when_it_is_true(self): + assert "identifier" not in glossary.dumps(make(term("list", "danh sách"))) + assert "identifier: true" in glossary.dumps(make(term("float", "số thực", identifier=True))) + def test_the_version_is_written(self): assert "version: 7\n" in glossary.dumps(make(version=7)) @@ -534,6 +561,7 @@ class TestRoundTrip: term("context manager", "trình quản lý ngữ cảnh"), term("decorator", "decorator", keep_en=True, note='the community keeps "decorator"'), term("list", "danh sách", context="library/stdtypes", note="a\\b"), + term("float", "số thực", identifier=True), term("no", "no", keep_en=True), term("3.15", "3.15", keep_en=True), ) @@ -597,6 +625,16 @@ def test_the_version_and_the_counts_are_stated(self): rendered = glossary.table(make(term("decorator", "decorator", keep_en=True), version=7)) assert "Version 7. 1 terms, 1 of them kept in English." in rendered + def test_a_standalone_row_says_so_where_a_reviewer_will_read_it(self): + rendered = glossary.table(make(term("float", "số thực", identifier=True))) + assert "| float | số thực | An entry that is only `float` names the thing" in rendered + + def test_a_kept_row_does_not_repeat_itself_in_the_note(self): + """``standalone`` holds every kept row, and saying so on each of the 52 + of them would push the useful notes off the side of the table.""" + rendered = glossary.table(make(term("sys", "sys", keep_en=True, identifier=True))) + assert "names the thing" not in rendered + def test_a_context_becomes_a_note_a_reviewer_can_act_on(self): rendered = glossary.table(make(term("list", "danh sách", context="stdtypes"))) assert "Only where the path or msgctxt contains `stdtypes`." in rendered diff --git a/tests/test_render.py b/tests/test_render.py index ac8e6f7..cc13218 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -53,6 +53,20 @@ def test_a_note_travels_with_the_row(self) -> None: term = Term(en="type", vi="type", keep_en=True, note="not kiểu in this sense") assert "not kiểu in this sense" in render.terminology([term]) + def test_a_row_that_stands_alone_says_both_things(self) -> None: + """``library/struct.po`` has "float" as a cell in the format-code table + two entries away from a sentence about floats, so a batch can hold both + readings and the rendering on its own would get the cell translated.""" + line = render.terminology([Term(en="float", vi="số thực", identifier=True)]) + assert line.startswith("- float -> số thực") + assert "leave it in English" in line + + def test_a_keep_en_row_does_not_repeat_itself(self) -> None: + """It already says to leave the English alone, and saying it twice on + one line is how a prompt starts reading as generated.""" + term = Term(en="sys", vi="sys", keep_en=True, identifier=True) + assert render.terminology([term]) == f"- sys -> {render.KEEP}" + def test_no_matching_row_says_so_rather_than_leaving_a_blank_heading(self) -> None: assert render.terminology([]) == render.NO_TERMS assert render.NO_TERMS in render.system(()) From 85450a0c93df38041e4985769ef87a6da77a6135 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:14:06 +0700 Subject: [PATCH 2/2] Lock the version bump --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 1caf606..468e0c4 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.10" +version = "0.1.11" source = { editable = "." } dependencies = [ { name = "httpx" },