From 73e1fdaae9ed586fd1f492b3c7050e39959d217d Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:02:24 +0700 Subject: [PATCH] Read three more shapes of code as literal blocks L01 reports an entry whose translation is identical to its source, and 5 of its 9 findings were entries there was nothing to translate in. The block rule called them prose, so each one went to a model, came back unchanged because unchanged was correct, and was then refused for being unchanged. Three shapes were missing. A block opener quoted without its body has no second line to indent and no call to recognise, which is `def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2):`. A line with a comment on the end is code plus English and the English hid the code, which is `parrot(1000) # 1 positional argument`. And a call whose arguments are quoted strings has spaces inside the quotes, and the spacing rule was counting those as prose, which is `parrot('a million', 'bereft of life', 'jump')`. The keyword list stops where English starts. `if`, `for`, `while`, `with` and `else` open blocks and are also ordinary words, and a colon is how the documentation introduces a list. Adding them reaches 4 more entries and 2 sentences, `while a positional argument could be created like::` and `if it is 3, implements::`, so they are out and there are tests holding them out. The comment rule wants two spaces before the hash, which is how PEP 8 says to write one and how every one of these in the corpus is written. 67 entries move from prose to literal_block, 0.08 per cent of the corpus. 67 is small enough to read one at a time and all 67 were read: no prose among them. One batch fewer on a full run. 5 of the 67 are entries somebody had already translated, and those 5 are exactly what L01 was reporting. They leave the memory, because human says who typed a string and not that the string wanted typing. --- pyproject.toml | 2 +- src/pydocvi/classify.py | 73 ++++++++++++++++++++++++++++++++- tests/test_classify.py | 89 +++++++++++++++++++++++++++++++++++++++++ tests/test_corpus.py | 16 ++++++-- uv.lock | 2 +- 5 files changed, 174 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e920261..bfbf159 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "python-docs-vi-translator" -version = "0.1.6" +version = "0.1.7" description = "Translates the CPython documentation into Vietnamese gettext catalogs, with a deterministic audit over every string it writes" readme = "README.md" license = "MIT" diff --git a/src/pydocvi/classify.py b/src/pydocvi/classify.py index 512b900..b13591c 100644 --- a/src/pydocvi/classify.py +++ b/src/pydocvi/classify.py @@ -60,6 +60,35 @@ _PATH = re.compile(r"^[\w.~-]+(?:[/\\][\w.~-]+)+$") _TIGHT = re.compile(r"^(?:[\w.]+\(.*\)|[\[{].*[\]}])$") +#: A line that opens an indented block, for the blocks whose body is not in the +#: entry. ``def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2):`` is a whole entry +#: in ``tutorial/controlflow.po`` and ``case (Point(x1, y1), Point(x2, y2) as +#: p2): ...`` is another, both quoted for their signature with the body left off, +#: so there is no second line to be indented and no call to recognise. +#: +#: The keyword list stops where English starts. ``if``, ``for``, ``while``, +#: ``with`` and ``else`` are the other block openers and all five are ordinary +#: words, and a colon is how the documentation introduces a list. Adding them +#: reaches four more entries and two of them are sentences: ``while a positional +#: argument could be created like::`` and ``if it is 3, implements::``. That is +#: the failure named at the top of this module, so the wide list was measured, +#: read and dropped. ``def``, ``class``, ``case``, ``match``, ``elif``, +#: ``except``, ``finally`` and ``try`` open no English sentence in this corpus. +_OPENER = re.compile( + r"^(?:async\s+)?(?:def|class|case|match|elif|except|finally|try)\b.*:(?:\s*\.\.\.)?$" +) + +#: An inline comment on the end of a line of code, and the code in front of it. +#: +#: Two spaces, which is how PEP 8 says to write one and how every one of these +#: in the corpus is written. One space would reach two more entries and read a +#: hash anywhere in a sentence as the start of a comment, and the hash is a +#: heading marker in more than one markup language. +_COMMENTED = re.compile(r"^(.*?\S)\s{2,}#") + +#: A quoted string, whose insides are data and not prose. +_QUOTED = re.compile(r"'[^']*'|\"[^\"]*\"") + #: A terminal prompt, with the virtualenv name some transcripts put in front of #: it. One of these anywhere in an entry makes the whole entry a transcript, #: because the lines around a prompt are the output of the command in it. @@ -200,6 +229,21 @@ def is_literal_block(msgid: str) -> bool: So a line that looks like code counts as well, and an entry counts when every line of it does. That adds 738 entries over the whole corpus, 0.85 per cent, and two random samples of them read as code with no prose in either. + + Three shapes of code were still missing from that, all found by reading + ``L01``, the check that reports an entry whose translation is its source. A + block opener quoted without its body has no second line to indent and no + call to recognise: ``def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2):``. A + line with a comment on the end is code plus English, and the English hid the + code: ``parrot(1000) # 1 positional argument``. And a call whose arguments + are quoted strings has spaces inside the quotes, which the spacing rule was + counting as prose: ``parrot('a million', 'bereft of life', 'jump')``. + + Together they move 67 entries, and 67 is a number small enough to read one + by one, which is what happened: no prose among them. Five of the 67 are + entries somebody had already translated, and all five are what ``L01`` was + reporting, translations identical to their source because there was nothing + to translate. The classifier now says so before a model is asked. """ lines = [line for line in msgid.split("\n") if line.strip()] if not lines: @@ -212,12 +256,30 @@ def is_literal_block(msgid: str) -> bool: def _is_code(line: str) -> bool: - """Whether one stripped line is code rather than a sentence.""" + """Whether one stripped line is code rather than a sentence. + + Asked twice where there is a comment on the end, once of the whole line and + once of the code in front of the hash. ``parrot('a million', 'bereft of + life', 'jump') # 3 positional arguments`` is a call, and none of the rules + below can see that while the English on the end is still attached. + """ + return _shaped(line) or _shaped(_uncommented(line)) + + +def _uncommented(line: str) -> str: + """The code in front of an inline comment, or the line if it has none.""" + found = _COMMENTED.match(line) + return found.group(1) if found else line + + +def _shaped(line: str) -> bool: + """Whether the line is code, taken as it is written.""" return bool( _SHEBANG.match(line) or _IMPORT.match(line) or _ASSIGNMENT.match(line) or _INVOCATION.match(line) + or _OPENER.match(line) or _PATH.match(line) or _tight(line) ) @@ -232,10 +294,17 @@ def _tight(line: str) -> bool: version of this rule sent both of them to passthrough. The thing that separates them is spacing: code puts a space after a comma and nowhere else, and English puts one between every pair of words. + + Which is true of code, and not of the strings inside it. ``parrot('a + million', 'bereft of life', 'jump')`` is a call whose arguments are English, + and counting the spaces in them read it as a sentence. So the quoted spans + are masked before the spacing is counted. What is inside them is data, and + this rule has no opinion about data. """ if not _TIGHT.match(line): return False - return all(line[i - 1] == "," for i, char in enumerate(line) if char == " " and i) + masked = _QUOTED.sub(lambda found: "_" * len(found.group(0)), line) + return all(masked[i - 1] == "," for i, char in enumerate(masked) if char == " " and i) def is_version_marker(msgid: str) -> bool: diff --git a/tests/test_classify.py b/tests/test_classify.py index 84179c1..8c506f9 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -149,6 +149,95 @@ def test_a_prompt_wearing_a_virtualenv_name(self) -> None: assert classify.is_literal_block(msgid) +class TestBlockOpeners: + """A signature quoted without the body it opens. + + There is no second line to be indented, no assignment and no call, so every + rule above this one reads them as a sentence ending in a colon. + """ + + @pytest.mark.parametrize( + "msgid", + [ + "def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2):", + "case (Point(x1, y1), Point(x2, y2) as p2): ...", + "class Bag[T]: ...", + "def func[T](arg: T): ...", + "async def f():", + "match command.split():", + "except* TypeError:", + ], + ) + def test_an_opener_with_no_body(self, msgid: str) -> None: + assert classify.classify(msgid) is Kind.LITERAL_BLOCK + + @pytest.mark.parametrize( + "msgid", + [ + "while a positional argument could be created like::", + "if it is 3, implements::", + "for example, the following two calls are equivalent::", + "with the following result:", + "else, the value is returned unchanged:", + ], + ) + def test_the_keywords_that_are_also_english(self, msgid: str) -> None: + """The list stops where English starts, and this is why. Adding ``if``, + ``for``, ``while``, ``with`` and ``else`` reaches four more entries and + two of these sentences, and a colon is how the documentation introduces + a list.""" + assert not classify.is_literal_block(msgid) + assert classify.classify(msgid).translatable + + def test_a_word_that_merely_starts_with_a_keyword(self) -> None: + """``\\b`` after the keyword, not before the rest of the line.""" + assert not classify.is_literal_block("classes and their attributes:") + + +class TestTrailingComments: + """Code with English on the end of it, which is most of a tutorial.""" + + @pytest.mark.parametrize( + "msgid", + [ + "parrot(1000) # 1 positional argument", + "parrot(voltage=1000) # 1 keyword argument", + "parrot() # required argument missing", + "parrot(actor='John Cleese') # unknown keyword argument", + "from . import echo # relative import", + ], + ) + def test_a_comment_does_not_hide_the_code(self, msgid: str) -> None: + assert classify.classify(msgid) is Kind.LITERAL_BLOCK + + def test_two_spaces_before_the_hash_not_one(self) -> None: + """Which is how PEP 8 says to write one and how every one of these in + the corpus is written. One space reads a hash anywhere in a sentence as + the start of a comment, and a hash opens a heading in more than one + markup language.""" + assert not classify.is_literal_block("Section # 3 (a heading)") + + def test_the_comment_alone_proves_nothing(self) -> None: + assert not classify.is_literal_block("Sorted by # of downloads") + + +class TestQuotedArguments: + def test_a_call_whose_arguments_are_sentences(self) -> None: + """The spacing rule reads a space that follows no comma as prose, and + the spaces here are inside string literals, where this rule has no + opinion about what the text says.""" + assert classify.is_literal_block("parrot('a million', 'bereft of life', 'jump')") + + def test_a_double_quoted_one(self) -> None: + msgid = 'Popen(["/usr/bin/git", "commit", "-m", "Fixes a bug."])' + assert classify.classify(msgid) is Kind.LITERAL_BLOCK + + def test_masking_does_not_excuse_the_spacing_outside_the_quotes(self) -> None: + """Only the insides are data. A call written with prose spacing between + its arguments is still not how anybody writes code.""" + assert not classify.is_literal_block("sorted() ('a builtin' in module builtins)") + + class TestVersionMarker: @pytest.mark.parametrize( "msgid", diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 490c5fa..7f055d4 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -35,7 +35,11 @@ #: Two constants where there was one, on purpose. The mirror's count is a fact #: about the mirror and this is a decision this tool makes, and folding them #: back together would hide the next change to either. -EXPECTED_HUMAN_SEGMENTS = 1_299 +#: 1 299 until the block rule learned to read an opener, a trailing comment and +#: a call holding quoted strings. The five it lost are the five ``L01`` was +#: reporting: ``def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2):`` and the rest, +#: translated by being copied, because copying was the correct answer. +EXPECTED_HUMAN_SEGMENTS = 1_294 #: Re-measured after the identifier rule was narrowed to need a dot, an #: underscore or a digit. 3 193 entries left ``version_marker``: 3 164 to prose @@ -49,14 +53,18 @@ #: as needing no translation. The classifier decides what a run costs and what #: it never looks at, and both are silent from any other angle, which is why #: these numbers are written down. +#: Re-measured again for the three block shapes the rule was missing. 67 entries +#: left prose, which is a cheap direction: they are code, they were being sent +#: to a model, and the model was returning them unchanged and being refused for +#: it. One batch fewer on a full run. EXPECTED_KINDS = { - "prose": 75_660, + "prose": 75_593, "noop": 5_721, "doctest": 2_121, - "literal_block": 2_856, + "literal_block": 2_923, "version_marker": 650, } -EXPECTED_BATCHES = 2_802 +EXPECTED_BATCHES = 2_801 EXPECTED_CHANGELOG_BATCHES = 576 EXPECTED_OVERSIZED = 6 EXPECTED_LARGEST_ENTRY = 12_707 diff --git a/uv.lock b/uv.lock index 74f9ebd..590b218 100644 --- a/uv.lock +++ b/uv.lock @@ -678,7 +678,7 @@ wheels = [ [[package]] name = "python-docs-vi-translator" -version = "0.1.5" +version = "0.1.7" source = { editable = "." } dependencies = [ { name = "httpx" },