diff --git a/pyproject.toml b/pyproject.toml index e535559..c6537ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.10" +version = "0.1.11" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/audit/language.py b/src/pydocvi/audit/language.py index a8ef928..375b708 100644 --- a/src/pydocvi/audit/language.py +++ b/src/pydocvi/audit/language.py @@ -155,27 +155,39 @@ def l02_not_the_source(corpus: Corpus) -> Iterator[Finding]: refusing the work or a classifier that should have called it passthrough, and both are worth a line in the report. - An entry whose whole ``msgid`` is a term the glossary keeps in English is - exempt too, and that exemption is the reason the glossary has a ``keep_en`` - field at all. Narrowing the identifier rule put 6 558 single-word entries - into :meth:`Corpus.prose` for the first time and took this check from 10 - findings to 144. A third of them were entries reading ``sys``, ``builtins``, - ``import``, ``exec``, ``NaN`` and ``Infinity``, all of which are index - entries naming a module or a statement, and all of which a reviewer left in - English because that is what a Vietnamese programmer calls them. + An entry whose whole ``msgid`` is a term the glossary marks as standing + alone is exempt too, and that exemption is the reason the glossary carries + the flag. Narrowing the identifier rule put 6 558 single-word entries into + :meth:`Corpus.prose` for the first time and took this check from 10 findings + to 144. A third of them were entries reading ``sys``, ``builtins``, + ``import``, ``exec`` and ``NaN``, all of which are index entries naming a + module or a statement, and all of which a reviewer left in English because + that is what a Vietnamese programmer calls them. Nothing in the string can tell those from ``module``, ``object`` and ``type``, which are the other 89 and are ordinary English words used as index categories. ``sys`` and ``Notes`` are the same shape, and that is the discrimination the classifier was narrowed for being unable to make. So it - is made once, by hand, in the glossary, where it is a written decision that - ``G03`` then checks in both directions rather than an exception buried here. - - Matched on the whole ``msgid`` and not on a substring. A kept term inside a + is made once, by hand, in the glossary, where it is a written decision + rather than an exception buried here. + + Read from :attr:`Glossary.standalone` and not from ``keep_en``, because the + two questions came apart. ``float`` is ``số thực`` in a sentence and the + name of a C type in the table of ``struct`` format codes, and while this + check read ``keep_en`` a row could only answer one of those. 69 of the 94 + findings ``G03`` was reporting were correct translations of ``type`` and + ``list``, held there by rows that said "keep this in English" when what they + meant was "leave the table cell alone". + + Matched on the whole ``msgid`` and not on a substring. A term inside a sentence says nothing about whether the sentence was translated, and this - check is about the entry. + check is about the entry. Nor is the match folded or de-inflected: ``Lists`` + is a section heading three times in the corpus, followed each time by prose + beginning "Lists are mutable sequences", and a heading is translated. """ - kept = {term.en for term in corpus.glossary.kept} if corpus.glossary is not None else set() + kept = ( + {term.en for term in corpus.glossary.standalone} if corpus.glossary is not None else set() + ) for one, entry in corpus.translated(): if classify.classify(entry.msgid) in PASSTHROUGH: continue diff --git a/src/pydocvi/glossary.py b/src/pydocvi/glossary.py index c4e40f4..9199581 100644 --- a/src/pydocvi/glossary.py +++ b/src/pydocvi/glossary.py @@ -52,6 +52,15 @@ class Term: such, and ``G02`` then checks the opposite thing: that the English survived rather than that it was replaced. + ``identifier`` is the field for the words that are both. "float" in a + sentence is "số thực" and the corpus says so twice; "float" on its own in the + table of struct format codes is the name of a C type and translating it would + break the table. One row has to be able to say both things, because the + alternative is what the file did before: pick the reading that suits the + louder check and be wrong about the other. ``keep_en`` answers the question + for running prose, ``identifier`` answers it for an entry that is nothing but + the term, and ``L02`` is the only check that asks the second one. + ``context`` is a filter, not prose. It is a fragment matched against the entry's ``msgctxt`` and against the file path, so a row carrying one applies only where that fragment appears. Prose about a row belongs in ``note``, @@ -62,6 +71,7 @@ class Term: en: str vi: str keep_en: bool = False + identifier: bool = False context: str | None = None note: str = "" @@ -123,6 +133,16 @@ def translated(self) -> tuple[Term, ...]: def kept(self) -> tuple[Term, ...]: return tuple(term for term in self.terms if term.keep_en) + @property + def standalone(self) -> tuple[Term, ...]: + """The rows an entry may equal and still be correct in English. + + A ``keep_en`` row is one of these without saying so: a term that stays + English everywhere stays English standing alone too. The rows that need + the flag are the ones translated in a sentence and named in a table. + """ + return tuple(term for term in self.terms if term.keep_en or term.identifier) + def with_terms(self, terms: Iterable[Term], *, version: int | None = None) -> Self: rows = tuple(terms) bumped = version if version is not None else self.version + bool(_changed(self.terms, rows)) @@ -344,7 +364,12 @@ def _differs(before: Term, after: Term) -> bool: ``note`` is not in here. A note is written for the person reading the file and changing one is not a reason to re-queue a few hundred entries. """ - return (before.vi, before.keep_en, before.context) != (after.vi, after.keep_en, after.context) + return (before.vi, before.keep_en, before.identifier, before.context) != ( + after.vi, + after.keep_en, + after.identifier, + after.context, + ) def _changed(before: Sequence[Term], after: Sequence[Term]) -> bool: @@ -500,7 +525,7 @@ def loads(text: str) -> Glossary: def _row(raw: object, at: int) -> Term: if not isinstance(raw, dict): raise GlossaryError(f"term {at} is not a mapping") - unknown = set(raw) - {"en", "vi", "keep_en", "context", "note"} + unknown = set(raw) - {"en", "vi", "keep_en", "identifier", "context", "note"} if unknown: raise GlossaryError(f"term {at} has unknown field(s): {', '.join(sorted(unknown))}") try: @@ -508,6 +533,7 @@ def _row(raw: object, at: int) -> Term: en=str(raw["en"]), vi=str(raw["vi"]), keep_en=bool(raw.get("keep_en", False)), + identifier=bool(raw.get("identifier", False)), context=str(raw["context"]) if raw.get("context") else None, note=str(raw.get("note", "")), ) @@ -537,6 +563,8 @@ def dumps(glossary: Glossary) -> str: out.append(f" vi: {scalar(term.vi)}") if term.keep_en: out.append(" keep_en: true") + if term.identifier: + out.append(" identifier: true") if term.context is not None: out.append(f" context: {scalar(term.context)}") if term.note: @@ -592,11 +620,20 @@ def table(glossary: Glossary) -> str: ] for term in match_order(glossary.terms): vi = f"`{term.en}` (kept)" if term.keep_en else term.vi - note = " ".join(part for part in (_context_note(term), term.note) if part) + note = " ".join( + part for part in (_context_note(term), _standalone_note(term), term.note) if part + ) lines.append(f"| {term.en} | {vi} | {_cell(note)} |") return "\n".join(lines) +def _standalone_note(term: Term) -> str: + """Said in the table because a reviewer reading one row cannot infer it.""" + if not term.identifier or term.keep_en: + return "" + return f"An entry that is only `{term.en}` names the thing and stays English." + + def _context_note(term: Term) -> str: return f"Only where the path or msgctxt contains `{term.context}`." if term.context else "" diff --git a/src/pydocvi/render.py b/src/pydocvi/render.py index a6ef63f..0bddff2 100644 --- a/src/pydocvi/render.py +++ b/src/pydocvi/render.py @@ -38,6 +38,13 @@ #: what to do rather than show a rendering that looks like a no-op. KEEP = "leave in English" +#: What an ``identifier`` row adds to its line. Such a row is translated in a +#: sentence and left alone when the whole string is the term, and a batch is +#: quite capable of holding both: ``library/struct.po`` has "float" as a cell in +#: the format-code table two entries away from a sentence about floats. Saying +#: only the rendering would get the cell translated. +ALONE = " (a string that is only this word names the thing: leave it in English)" + #: The first line of the user message names the file and what kind of writing it #: is, because the register is not a detail. The tutorial is addressed to a #: beginner and the C API reference is addressed to somebody writing an @@ -134,6 +141,7 @@ def terminology(terms: Sequence[Term]) -> str: return NO_TERMS return "\n".join( f"- {term.en} -> {KEEP if term.keep_en else term.vi}" + + (ALONE if term.identifier and not term.keep_en else "") + (f" ({term.note})" if term.note else "") for term in terms ) diff --git a/tests/test_audit_language.py b/tests/test_audit_language.py index 4f7ffae..0b1348e 100644 --- a/tests/test_audit_language.py +++ b/tests/test_audit_language.py @@ -24,6 +24,12 @@ def with_kept(corpus: object, *english: str) -> Corpus: return replace(corpus, glossary=Glossary(version=1, terms=terms)) # type: ignore[type-var] +def with_standalone(corpus: object, en: str, vi: str) -> Corpus: + """The same corpus with one row translated in prose and kept on its own.""" + terms = (Term(en=en, vi=vi, identifier=True),) + return replace(corpus, glossary=Glossary(version=1, terms=terms)) # type: ignore[type-var] + + class TestL01: LONG = "A sentence long enough that a reader would expect a diacritic in it." @@ -78,9 +84,24 @@ def test_a_kept_term_inside_a_sentence_exempts_nothing(self) -> None: assert len(findings(language.l02_not_the_source, corpus)) == 1 def test_a_word_the_glossary_does_not_keep_is_still_reported(self) -> None: - """``module`` is the other side of the same 144 findings, an ordinary - English word used as an index category, and it wants translating.""" - corpus = with_kept(over("module", "module", flags=()), "sys") + """``object`` is the other side of the same 144 findings, an ordinary + English word used as an index category, and it wants translating. It is + one of 36 such entries and the 124 sentences around them say ``đối + tượng`` 93 times.""" + corpus = with_kept(over("object", "object", flags=()), "sys") + assert len(findings(language.l02_not_the_source, corpus)) == 1 + + def test_a_row_translated_in_prose_can_still_stand_alone(self) -> None: + """``float`` is ``số thực`` in a sentence and the name of a C type in + the ``struct`` format table. While this check read ``keep_en`` the row + could say one or the other, and saying "keep the English" to get the + table cell right made ``G03`` report both correct translations of it.""" + corpus = with_standalone(over("float", "float", flags=()), "float", "số thực") + assert findings(language.l02_not_the_source, corpus) == [] + + def test_standing_alone_does_not_excuse_a_sentence(self) -> None: + english = "Divide and get a float." + corpus = with_standalone(over(english, english), "float", "số thực") assert len(findings(language.l02_not_the_source, corpus)) == 1 def test_no_glossary_means_no_exemption_rather_than_a_crash(self) -> None: diff --git a/tests/test_glossary.py b/tests/test_glossary.py index eddc4d9..8a634e9 100644 --- a/tests/test_glossary.py +++ b/tests/test_glossary.py @@ -75,6 +75,20 @@ def test_kept_and_translated_rows_are_separable(self): assert [row.en for row in rows.kept] == ["decorator"] assert [row.en for row in rows.translated] == ["list"] + def test_a_kept_row_stands_alone_without_being_told_to(self): + """A term that is English in every sentence is English on its own too, + so the flag is only ever needed by rows that carry a rendering.""" + rows = make(term("decorator", "decorator", keep_en=True), term("list", "danh sách")) + assert [row.en for row in rows.standalone] == ["decorator"] + + def test_a_translated_row_can_still_stand_alone(self): + """The case the field was added for. ``float`` is ``số thực`` in a + sentence and the name of a C type in the ``struct`` format table.""" + rows = make(term("float", "số thực", identifier=True), term("list", "danh sách")) + assert [row.en for row in rows.standalone] == ["float"] + assert [row.en for row in rows.translated] == ["float", "list"] + assert rows.kept == () + def test_len_counts_rows(self): assert len(make(*NESTED)) == 2 @@ -287,6 +301,13 @@ def test_flipping_keep_en_is_a_change(self): after = make(term("decorator", "decorator", keep_en=True), version=2) assert len(glossary.diff(before, after).changed) == 1 + def test_flipping_identifier_is_a_change_too(self): + """It moves entries in and out of ``L02`` and changes what the prompt + says, so a run made before it is not a run made after it.""" + before = make(term("float", "số thực")) + after = make(term("float", "số thực", identifier=True), version=2) + assert len(glossary.diff(before, after).changed) == 1 + def test_adding_a_context_is_a_change(self): before = make(term("list", "danh sách")) after = make(term("list", "danh sách", context="stdtypes"), version=2) @@ -451,10 +472,12 @@ def test_every_field_survives(self): text = ( "version: 1\nterms:\n" ' - en: "decorator"\n vi: "decorator"\n keep_en: true\n' + " identifier: true\n" ' context: "library"\n note: "the community keeps the English"\n' ) row = glossary.loads(text).terms[0] - assert (row.keep_en, row.context, row.note) == ( + assert (row.keep_en, row.identifier, row.context, row.note) == ( + True, True, "library", "the community keeps the English", @@ -513,6 +536,10 @@ def test_keep_en_is_written_only_when_it_is_true(self): assert "keep_en" not in glossary.dumps(make(term("list", "danh sách"))) assert "keep_en: true" in glossary.dumps(make(term("decorator", "decorator", keep_en=True))) + def test_identifier_is_written_only_when_it_is_true(self): + assert "identifier" not in glossary.dumps(make(term("list", "danh sách"))) + assert "identifier: true" in glossary.dumps(make(term("float", "số thực", identifier=True))) + def test_the_version_is_written(self): assert "version: 7\n" in glossary.dumps(make(version=7)) @@ -534,6 +561,7 @@ class TestRoundTrip: term("context manager", "trình quản lý ngữ cảnh"), term("decorator", "decorator", keep_en=True, note='the community keeps "decorator"'), term("list", "danh sách", context="library/stdtypes", note="a\\b"), + term("float", "số thực", identifier=True), term("no", "no", keep_en=True), term("3.15", "3.15", keep_en=True), ) @@ -597,6 +625,16 @@ def test_the_version_and_the_counts_are_stated(self): rendered = glossary.table(make(term("decorator", "decorator", keep_en=True), version=7)) assert "Version 7. 1 terms, 1 of them kept in English." in rendered + def test_a_standalone_row_says_so_where_a_reviewer_will_read_it(self): + rendered = glossary.table(make(term("float", "số thực", identifier=True))) + assert "| float | số thực | An entry that is only `float` names the thing" in rendered + + def test_a_kept_row_does_not_repeat_itself_in_the_note(self): + """``standalone`` holds every kept row, and saying so on each of the 52 + of them would push the useful notes off the side of the table.""" + rendered = glossary.table(make(term("sys", "sys", keep_en=True, identifier=True))) + assert "names the thing" not in rendered + def test_a_context_becomes_a_note_a_reviewer_can_act_on(self): rendered = glossary.table(make(term("list", "danh sách", context="stdtypes"))) assert "Only where the path or msgctxt contains `stdtypes`." in rendered diff --git a/tests/test_render.py b/tests/test_render.py index ac8e6f7..cc13218 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -53,6 +53,20 @@ def test_a_note_travels_with_the_row(self) -> None: term = Term(en="type", vi="type", keep_en=True, note="not kiểu in this sense") assert "not kiểu in this sense" in render.terminology([term]) + def test_a_row_that_stands_alone_says_both_things(self) -> None: + """``library/struct.po`` has "float" as a cell in the format-code table + two entries away from a sentence about floats, so a batch can hold both + readings and the rendering on its own would get the cell translated.""" + line = render.terminology([Term(en="float", vi="số thực", identifier=True)]) + assert line.startswith("- float -> số thực") + assert "leave it in English" in line + + def test_a_keep_en_row_does_not_repeat_itself(self) -> None: + """It already says to leave the English alone, and saying it twice on + one line is how a prompt starts reading as generated.""" + term = Term(en="sys", vi="sys", keep_en=True, identifier=True) + assert render.terminology([term]) == f"- sys -> {render.KEEP}" + def test_no_matching_row_says_so_rather_than_leaving_a_blank_heading(self) -> None: assert render.terminology([]) == render.NO_TERMS assert render.NO_TERMS in render.system(()) diff --git a/uv.lock b/uv.lock index 1caf606..468e0c4 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.10" +version = "0.1.11" source = { editable = "." } dependencies = [ { name = "httpx" },