Strip docx underline spans and flag the minus signs OCR reads from separator lines

docs/plan.md

@@ -84,9 +84,11 @@ detects a question or part that one route missed. The adjudication resolves word
 two routes read differently.
 
 The tiers do not detect an error in the markdown. Both routes read the same OCR output, so
-a word Mathpix misread, or a separator line Mathpix read as a minus sign, passes every
-tier. The comparison with an exported set, or a reader, detects those. An OCR check is
-separate work.
+a word Mathpix misread passes every tier. The comparison with an exported set, or a
+reader, detects it. An OCR check is separate work. The one misread the route detects is a
+separator line read as a minus sign: `stray_minus` flags a field whose display maths
+begins or ends with a minus sign, and a field holding a minus sign on a line of its own
+beside a display maths.
 
 ## Response areas
 

in2lambda_agent/routes.py

@@ -6,7 +6,9 @@
 quote of the markdown (`not_verbatim`). The two replies are compared field by field
 (`disputed`); a disputed field goes to a small second call that may pick one side or a
 passage of the source, never its own words (`adjudicate`); what neither settles is a flag
-for a person (`reconcile`). `to_set` and `build` write the result with in2lambda.
+for a person (`reconcile`). A minus sign inside or beside a display maths, which Mathpix
+reads from a separator line, is flagged too (`stray_minus`). `to_set` and `build` write
+the result with in2lambda.
 
 A reply is a list of questions: {"title", "main_text", "parts": [{"content",
 "options", "answer", "worked_solution"}]}. Field keys are 1-based: `q2.p1.content`.
@@ -32,6 +34,8 @@
 
 TEXT_FIELDS = ("content", "answer", "worked_solution")
 
+STRAY_MINUS = "a stray minus sign inside or beside a display maths; Mathpix reads a separator line as one"
+
 _FOLDS = (
     ("\\left(", "("), ("\\right)", ")"), ("\\left[", "["), ("\\right]", "]"),
     ("\\mathrm{~", "\\mathrm{"), ("\\text {", "\\text{"), ("\\space", " "),
@@ -87,6 +91,29 @@ def not_verbatim(reply: Reply_, source: str) -> list[str]:
     return found
 
 
+# A minus sign on a line of its own, after a $$ line or before one, blank lines between.
+# Mathpix reads a separator line of the printed page either into the display maths beside
+# it or as a paragraph of its own, so both forms are stray.
+_LONE_MINUS = re.compile(
+    r"\$\$[ \t]*\n(?:[ \t]*\n)*[ \t]*-[ \t]*(?:\n|\Z)"
+    r"|(?:\A|\n)[ \t]*-[ \t]*\n(?:[ \t]*\n)*[ \t]*\$\$"
+)
+
+
+def stray_minus(reply: Reply_) -> list[str]:
+    """The fields holding a minus sign Mathpix read from a separator line.
+
+    A display maths begins or ends with the minus sign, or the minus sign stands on a
+    line of its own beside the block.
+    """
+    found = []
+    for key, text in fields(reply).items():
+        blocks = [b.strip() for b in re.findall(r"\$\$(.*?)\$\$", text or "", re.S)]
+        if any(b.startswith("-") or b.endswith("-") for b in blocks) or _LONE_MINUS.search(text or ""):
+            found.append(key)
+    return found
+
+
 def disputed(a: Reply_, b: Reply_) -> list[str]:
     """Where two replies differ: a question or part one lacks, or a field worded differently."""
     found: list[str] = []
@@ -294,6 +321,9 @@ class Converted:
     tokens: int = 0
 
 
+_UNDERLINE = Path(__file__).parent / "underline.lua"
+
+
 def markdown_of(document: Path, cache_dir: Path, settings: Settings) -> tuple[str, Path]:
     """The document as markdown, and the folder its images are in."""
     document = Path(document)
@@ -305,7 +335,15 @@ def markdown_of(document: Path, cache_dir: Path, settings: Settings) -> tuple[st
         return ocr.markdown.read_text(encoding="utf-8"), ocr.markdown.parent
     if document.suffix.lower() in (".md", ".markdown"):
         return document.read_text(encoding="utf-8"), document.parent
-    out = subprocess.run(["pandoc", str(document), "-t", "commonmark_x", "--wrap=none"], capture_output=True, check=True)
+    # An underlined run of a docx, and \underline{} of a tex file, is written by
+    # commonmark_x as [text]{.underline}, which Lambda Feedback does not render. The
+    # filter drops the underline and keeps the words. Turning bracketed_spans off instead
+    # writes the run as raw HTML, and turning raw_html off with it drops every table
+    # commonmark_x cannot write as a pipe table.
+    out = subprocess.run(
+        ["pandoc", str(document), "-t", "commonmark_x", "--wrap=none", "--lua-filter", str(_UNDERLINE)],
+        capture_output=True, check=True,
+    )
     return out.stdout.decode("utf-8"), document.parent
 
 
@@ -334,6 +372,9 @@ def convert(
         reply, flags = reconciled.fields, reconciled.flags
     else:
         flags = [Flag(k, fields(reply)[k], "", "not a quote of the source") for k in not_verbatim(reply, source)]
+    for k in stray_minus(reply):
+        if not any(f.field == k for f in flags):
+            flags.append(Flag(k, fields(reply)[k], "", STRAY_MINUS))
     built = to_set(reply, name=name, directory=images)
     return Converted(set=built, zip_path=build(built, out_dir), flags=flags, reply=reply, tokens=tokens)
 

in2lambda_agent/underline.lua

@@ -0,0 +1,5 @@
+-- Keep the words of an underlined run and drop the underline, which Lambda Feedback
+-- renders in no form pandoc can write. See markdown_of in routes.py.
+function Underline(el)
+  return el.content
+end

tests/test_routes.py

@@ -21,6 +21,7 @@
 from conftest import FakeBackend
 
 import in2lambda_agent.routes as routes
+from in2lambda_agent.settings import Settings
 
 ME2 = Path(__file__).parent / "fixtures" / "me2"
 QUESTIONS = (ME2 / "questions.md").read_text()
@@ -64,6 +65,78 @@ def test_an_empty_field_is_not_a_quote_of_anything_and_is_not_flagged():
     assert routes.not_verbatim(empty, QUESTIONS + "\n" + SOLUTIONS) == []
 
 
+# --- the document as markdown -----------------------------------------------------------
+
+
+@pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc")
+def test_an_underlined_run_of_a_docx_is_written_without_a_bracketed_span(tmp_path):
+    # The table has a cell of two paragraphs, which commonmark_x cannot write as a pipe
+    # table and so writes as raw HTML: a conversion that dropped raw HTML to be rid of
+    # the span would write [TABLE] here instead of the numbers.
+    source = tmp_path / "sheet.md"
+    source.write_text(
+        "Find [the mass]{.underline} of the piston.\n\n"
+        "+-----------+-----------+\n"
+        "| Stress    | Strain    |\n"
+        "+===========+===========+\n"
+        "| 120 MPa   | 0.8%      |\n"
+        "|           |           |\n"
+        "| at 400 °C | in 1000 h |\n"
+        "+-----------+-----------+\n"
+    )
+    docx = tmp_path / "sheet.docx"
+    subprocess.run(["pandoc", str(source), "-f", "markdown", "-o", str(docx)], check=True)
+    markdown, _ = routes.markdown_of(docx, tmp_path, Settings())
+    assert "Find the mass of the piston." in markdown
+    assert "{.underline}" not in markdown
+    assert "<u>" not in markdown and "<span" not in markdown
+    for cell in ("Stress", "Strain", "120 MPa", "0.8%", "at 400 °C", "in 1000 h"):
+        assert cell in markdown
+
+
+# --- a display maths that begins or ends with a minus sign --------------------------------
+
+
+def test_the_me2_worked_solutions_with_separator_minus_signs_are_named():
+    assert routes.stray_minus(REPLY) == ["q2.p1.worked_solution", "q3.p1.worked_solution"]
+
+
+def test_a_minus_on_a_line_of_its_own_beside_a_display_maths_is_stray():
+    reply = [
+        {
+            "title": "",
+            "main_text": "The mass entering is:\n-\n\n$$\nm = \\rho U A\n$$\n\n- \n\nwhere $A$ is the area.",
+            "parts": [],
+        }
+    ]
+    assert routes.stray_minus(reply) == ["q1.main_text"]
+
+
+def test_a_minus_inside_the_maths_or_inline_is_not_stray():
+    reply = [
+        {
+            "title": "",
+            "main_text": "A difference $$ a-b $$ and an inline $-x$.",
+            "parts": [{"content": "- $$\nx = 1\n$$", "options": [], "answer": "", "worked_solution": ""}],
+        }
+    ]
+    assert routes.stray_minus(reply) == []
+
+
+def test_convert_reports_the_stray_minus_as_a_flag(tmp_path):
+    result = routes.convert(
+        ME2 / "questions.md",
+        solutions=ME2 / "solutions.md",
+        out_dir=tmp_path / "out",
+        backend=FakeBackend(json.dumps(REPLY)),
+        settings=Settings(),
+    )
+    assert [(f.field, f.reason) for f in result.flags] == [
+        ("q2.p1.worked_solution", routes.STRAY_MINUS),
+        ("q3.p1.worked_solution", routes.STRAY_MINUS),
+    ]
+
+
 # --- tier 1: agreement ----------------------------------------------------------------
 
 
@@ -186,5 +259,10 @@ def test_the_me2_pair_converts_with_no_flag(tmp_path):
     (pdf,) = [p for p in target.glob("*.pdf") if "solutions" not in p.name]
     (solutions,) = target.glob("*solutions.pdf")
     result = routes.convert(pdf, solutions=solutions, out_dir=tmp_path / "out")
-    assert result.flags == []
+    # The printed solutions PDF holds separator lines that Mathpix reads as minus signs,
+    # so the worked solutions of Friction on a plate and Towing a submarine are flagged.
+    assert [(f.field, f.reason) for f in result.flags] == [
+        ("q2.p1.worked_solution", routes.STRAY_MINUS),
+        ("q3.p1.worked_solution", routes.STRAY_MINUS),
+    ]
     assert [q.title for q in result.set.questions] == [q["title"] for q in exported()]