diff --git a/docs/how-it-works.md b/docs/how-it-works.md index c8f9d69..dcbcf77 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -113,9 +113,10 @@ The stage prints one of three messages: and the time is the wall time of the call to one decimal place. The try number counts from 1 to `--tries`. * `kept try 2 of 3` — the loop wrote more than one spec, and this names the try saved - for the set: the one that left the fewest blocks unassigned and the fewest errors, - over this source and over the set's other document. The loop writes one spec where - the first leaves neither, and prints no `kept` line. + for the set: the one that scored lowest. A try's score adds up the blocks it left + unassigned, the errors the checks then found, the images it dropped and the blocks + it left unassigned in the set's other document. The loop writes one spec where the + first scores zero, and prints no `kept` line. in2lambda refuses a spec it cannot run, and the run raises `SpecRejected`. A spec this run wrote is deleted before that refusal reaches the user, and a spec this run wrote @@ -137,6 +138,19 @@ stage prints `no fields` where the spec wrote none. `4 ignored` is the number of blocks the spec's `ignore` selector matched. The unassigned blocks are listed by id, and the stage prints `none unassigned` where every block reached a field. +A spec that dropped an image adds a second half to the line, after the unassigned +blocks: `; 2 images dropped: b10 (lines 29-30), b14 (lines 41-42)`, or `; 1 image +dropped: b4 (lines 7-8)` for one. A dropped image is a block the spec marked ignore +whose lines hold a `![`. Mathpix writes a figure as a paragraph of its own — the image +line and then its caption — so an `ignore` selector matching the caption drops the +figure, and the set is built without it. in2lambda's checks report nothing about an +ignored block, so `package.ignored_images` reads the ignored blocks back and the half +names each dropped image by its block and its lines. + +Each dropped image counts toward the try's score, so a spec that drops one is written +again. Where the next spec drops the image too, the coverage line of the try the run +kept names it, and the run goes on to `validate` and builds the set without it. + ### `replay` The stage runs the commands an earlier run's fixing rounds ran, read from the file the @@ -193,9 +207,12 @@ prints one of six messages: * `b13 (lines 21-21) is in no field and not marked ignore.` — the errors, joined with `; `. A fixing round follows where `--rounds` is 1 or more. Under `--rounds 0` the run ends on this line, with no `fix` line after it. -* `ERRORS — writing the set's spec again` — the run reused a saved spec, the checks - fault the draft, and `--rounds` is 1 or more. The run writes the spec again and - prints `freeze`, `spec`, `coverage` and `validate` a second time. +* `ERRORS — writing the set's spec again` — the run reused a saved spec, `--rounds` is + 1 or more, and the checks fault the draft or the spec run dropped an image. The + errors are joined with `; `, and the message of each dropped image follows them, so a + reused spec whose one fault is `b4 (lines 7-8) holds an image and is marked ignore.` + says that alone. The run writes the spec again and prints `freeze`, `spec`, + `coverage` and `validate` a second time. * `ERRORS — left by round 2, no zip` — round 2 answered no error it was given, so the run ends with those errors and writes no zip. * `ERRORS — round limit 3 reached, no zip` — the last round of `--rounds` ran and the @@ -288,7 +305,7 @@ Each backend limits a call differently: | Call | What it is given | What it may write | | --- | --- | --- | -| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds and the blocks that spec left in no field in the set's other document | `in2lambda-spec.yaml`, and nothing else | +| Spec | the spec system prompt, the frozen source as `in2lambda.source.show` prints it, and, from the second call on, the spec before it, that spec's coverage line, the errors the report holds, the images that spec dropped and the blocks that spec left in no field in the set's other document | `in2lambda-spec.yaml`, and nothing else | | Spec rewrite | the same, with the saved spec and what running it covered as the first call's try 0 | `in2lambda-spec.yaml`, and nothing else | | Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the eight draft commands, and nothing else | @@ -298,15 +315,15 @@ mapping, and a reply naming a `layout` outside `PartsSepSol`, `PartsOneSol`, `PartSolPartSol` and `PartPartSolSol`. A run makes the spec call up to `--tries` times, three by default. Each call after the -first is asked for a spec that leaves fewer blocks unassigned and fewer errors behind -than the one before it, over this source and over the set's other document. The run -makes no further call once a spec leaves no block unassigned and no error behind, and -saves the try that left the fewest of both. +first is asked for a spec that leaves fewer blocks unassigned, fewer images ignored and +fewer errors behind than the one before it, over this source and over the set's other +document. The run makes no further call once a spec scores zero, and saves the try that +scored lowest. The spec rewrite is the same loop, with the saved spec and what running it covered as -try 0. It runs where the run reused a saved spec and the checks fault the draft, before -any fixing round, and takes `--tries` calls like any other spec. It writes layer 1 -fields, and it is not one of the `--rounds`. +try 0. It runs where the run reused a saved spec and the checks fault the draft or the +spec run dropped an image, before any fixing round, and takes `--tries` calls like any +other spec. It writes layer 1 fields, and it is not one of the `--rounds`. The fixing round's tools are the eight in2lambda draft commands: `mark ignore`, `question add`, `part add`, `question solution`, `part solution`, `field replace`, diff --git a/in2lambda_agent/package.py b/in2lambda_agent/package.py index 2898f3c..a9d680a 100644 --- a/in2lambda_agent/package.py +++ b/in2lambda_agent/package.py @@ -91,6 +91,9 @@ class Coverage: fields: How many fields each layer wrote, keyed by layer number. ignored: How many blocks the spec's `ignore` selector matched. unassigned: The ids of blocks in no field and not ignored. + dropped: One finding per ignored block whose lines hold an image, as + `ignored_images` reports them. The set is built without those + images, and nothing in2lambda checks says so. """ layout: str @@ -98,6 +101,7 @@ class Coverage: fields: dict[int, int] = field(default_factory=dict) ignored: int = 0 unassigned: list[str] = field(default_factory=list) + dropped: list["Finding"] = field(default_factory=list) def __str__(self) -> str: """The one line the coverage stage prints.""" @@ -106,10 +110,15 @@ def __str__(self) -> str: for layer, count in sorted(self.fields.items()) ) left = ", ".join(self.unassigned) if self.unassigned else "none" - return ( + line = ( f"{self.layout}: {self.blocks} blocks, {written or 'no fields'}, " f"{self.ignored} ignored, {left} unassigned" ) + if self.dropped: + images = ", ".join(_where(one.field, one.ranges) for one in self.dropped) + plural = "image" if len(self.dropped) == 1 else "images" + line += f"; {len(self.dropped)} {plural} dropped: {images}" + return line @dataclass @@ -280,9 +289,88 @@ def spec_run(draft: Path, spec: Path) -> Coverage: coverage.unassigned = [ finding["field"] for finding in in2lambda.draft.report.uncovered(found) ] + coverage.dropped = ignored_images(draft) return coverage +def ignored_images(draft: Path) -> list[Finding]: + """The blocks a spec marked ignore whose lines hold an image. + + A figure belongs to the question or part it illustrates. Mathpix writes a + figure as one paragraph, the image line and then its caption, so an `ignore` + selector matching the caption's `Figure n:` marks the image ignored and the + set is built without it. in2lambda's checks report nothing about an ignored + block, so the agent reads the ignored blocks back and reports each image. + + Args: + draft: The draft file, after a spec has been run over it. + + Returns: + One finding per ignored block whose lines hold a markdown image, in the + order the blocks appear in the sources. The message follows the wording + of in2lambda's own coverage findings, so that a spec-writing prompt + reads the same for either finding. A source whose bytes are not text — + a docx, frozen as itself — has no such block to report. + """ + found = _frozen(draft) + read: dict[int, Optional[list[str]]] = {} + + def source_lines(number: int) -> Optional[list[str]]: + """The lines of one source, read the first time a block of it is ignored.""" + if number not in read: + path = Path(draft).parent / found["sources"][number]["source"] + read[number] = _source_lines(path) + return read[number] + + dropped = [] + for key, written in found["fields"].items(): + if not key.endswith(".ignore"): + continue + block = key[: -len(".ignore")] + # `2/b3` is the second source's block; `b3` is the first source's. + number, _, _ = block.rpartition("/") + lines = source_lines(int(number) - 1 if number else 0) + if lines is None: + continue + ranges = written["ranges"] + held = "\n".join("\n".join(lines[start - 1 : end]) for start, end in ranges) + if "![" in held: + dropped.append( + Finding( + check="coverage", + level=ERROR, + field=block, + ranges=ranges, + message=f"{_where(block, ranges)} holds an image and is " + "marked ignore.", + ) + ) + return sorted(dropped, key=lambda one: (one.field.rpartition("/")[0], one.ranges)) + + +def _where(field: str, ranges: list[list[int]]) -> str: + """A block and the lines it covers, as a finding's message names one. + + Returns `b10 (lines 29-30)`, which is how in2lambda's own findings name one. + """ + covered = ", ".join(f"{start}-{end}" for start, end in ranges) + return f"{field} (lines {covered})" + + +def _source_lines(path: Path) -> Optional[list[str]]: + """A frozen source read as text, or None where its bytes are not text. + + A docx is frozen as itself and is a zip, and a tex sheet of a real set need + not be UTF-8, so a source is decoded the way `is_document` decodes one and + a source holding a NUL byte is left alone: it has no line of markdown to + find an image in. + """ + raw = path.read_bytes() + if b"\x00" in raw: + return None + return raw.decode("utf-8", errors="replace").splitlines() + + def blocks(draft: Path) -> int: """How many blocks a draft's frozen source has. diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index 43b3476..546c992 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -273,7 +273,11 @@ def run( result.add_stage("replay", f"{ran} commands from {commands}") report = package.validate(draft) - if report.clean or rounds < 1: + # An image the saved spec marked ignore is a fault of the spec that + # in2lambda's checks say nothing about, so it sends the run into the + # rewrite loop as a faulted draft does. + dropped = [one.message for one in result.coverage.dropped] + if (report.clean and not dropped) or rounds < 1: result.add_stage( "validate", package.said(report) if report.clean else "; ".join(report.errors), @@ -281,7 +285,7 @@ def run( else: result.add_stage( "validate", - "; ".join(report.errors) + " — writing the set's spec again", + "; ".join(report.errors + dropped) + " — writing the set's spec again", ) previous = Previous( text=saved.read_text(encoding="utf-8"), diff --git a/in2lambda_agent/review.py b/in2lambda_agent/review.py index 89a2b88..989aff8 100644 --- a/in2lambda_agent/review.py +++ b/in2lambda_agent/review.py @@ -267,6 +267,7 @@ def _try_json(one: SpecTry) -> dict[str, Any]: "usage": asdict(one.usage), "unassigned": one.unassigned, "errors": one.errors, + "dropped": one.dropped, "second": one.second, "chosen": one.chosen, } @@ -279,6 +280,7 @@ def _try_from(saved: dict[str, Any]) -> SpecTry: usage=Usage(**saved["usage"]), unassigned=saved["unassigned"], errors=saved["errors"], + dropped=saved["dropped"], second=saved["second"], chosen=saved["chosen"], ) diff --git a/in2lambda_agent/spec.py b/in2lambda_agent/spec.py index 180fc9c..e0045c8 100644 --- a/in2lambda_agent/spec.py +++ b/in2lambda_agent/spec.py @@ -15,7 +15,8 @@ written against what running it covers rather than blind. `iterate_spec` writes one, runs it over this source and over another document of the set, reads the coverage and the validation report back to the next call, and saves the spec -that left the fewest blocks unassigned and the fewest errors behind. +that left the fewest blocks unassigned, the fewest images ignored and the +fewest errors behind. """ import json @@ -80,6 +81,9 @@ marker is still in the field's value, so `strip` is what takes it off. * A block indented under a list item is inside it, not beside it: such a question and its parts are one block, and there is nothing to select. + * A paragraph holding an image — `![...](...)` — is content, never `ignore`. + The figure belongs to the question or the solution it stands with, and a + set built from an ignored figure has lost it. A draft holds two documents where the solutions are written as a file of their own. The questions file is the first source, with block ids `b1` onwards, and @@ -141,6 +145,7 @@ class SpecTry: usage: What the call cost, all zeroes for try 0. unassigned: How many blocks the spec left in no field and not ignored. errors: How many errors the checks then found in the draft it filled. + dropped: How many blocks the spec marked ignore that hold an image. second: How many blocks the spec left in no field in another document of the set, or None where the run ran no spec over another document — the record's `second` says why. A spec in2lambda refuses over @@ -153,6 +158,7 @@ class SpecTry: usage: Usage = field(default_factory=Usage) unassigned: int = 0 errors: int = 0 + dropped: int = 0 second: Optional[int] = None chosen: bool = False @@ -163,8 +169,12 @@ def score(self) -> int: A block in no field is an error of the report as well as a line of the coverage, so it counts twice. That is the same double for every try and does not change the order they come in. + + The third term is the images the spec dropped. in2lambda's checks say + nothing about an ignored block, so a spec that ignores a figure is + scored like one that leaves a block unassigned and is written again. """ - return self.unassigned + self.errors + (self.second or 0) + return self.unassigned + self.errors + self.dropped + (self.second or 0) @dataclass @@ -295,10 +305,14 @@ def _revision(previous: Previous) -> str: said = [f"\nYour last spec for this set was:\n\n{previous.text}"] if previous.coverage is not None: said.append(f"\nRunning it over this source covered:\n\n{previous.coverage}\n") - if previous.report is not None and previous.report.errors: - said.append( - "\nThe checks then found:\n\n" + "\n".join(previous.report.errors) + "\n" - ) + # The images the spec dropped go in beside the report's errors, under the + # one heading: an ignored figure is a fault of the selectors like a block + # left in no field, and the next call answers both the same way. + found = list(previous.report.errors) if previous.report is not None else [] + if previous.coverage is not None: + found += [one.message for one in previous.coverage.dropped] + if found: + said.append("\nThe checks then found:\n\n" + "\n".join(found) + "\n") if previous.second is not None: left = ", ".join(previous.second.unassigned) or "no blocks" said.append( @@ -306,8 +320,9 @@ def _revision(previous: Previous) -> str: f"set, left {left} in no field.\n" ) said.append( - "\nWrite a spec that leaves fewer blocks unassigned and fewer errors " - "behind, over this source and over the rest of the set.\n" + "\nWrite a spec that leaves fewer blocks unassigned, fewer images " + "ignored and fewer errors behind, over this source and over the rest " + "of the set.\n" ) return "".join(said) @@ -328,9 +343,9 @@ def iterate_spec( """Writes the set's spec up to `tries` times and saves the best of them. Each call after the first is shown the spec before it, the coverage line, - the errors the checks found and the blocks the spec left over in another - document of the set. The loop stops at a spec that leaves no block - unassigned and no error behind, since a further call has nothing to improve. + the errors the checks found, the images the spec marked ignore and the + blocks the spec left over in another document of the set. The loop stops at + a spec that scores zero, since a further call has nothing to improve. Args: frozen: The markdown, tex or docx file each spec is run over. @@ -388,6 +403,7 @@ def iterate_spec( number=0, unassigned=len(previous.coverage.unassigned), errors=len(previous.report.errors), + dropped=len(previous.coverage.dropped), second=left_over, ) ) @@ -428,6 +444,7 @@ def iterate_spec( usage=reply.usage, unassigned=len(coverage.unassigned), errors=len(report.errors), + dropped=len(coverage.dropped), second=left_over, ) made.append(one) @@ -584,6 +601,7 @@ def record_run( "seconds": round(one.usage.seconds, 3), "unassigned": one.unassigned, "errors": one.errors, + "dropped": one.dropped, "second": one.second, "chosen": one.chosen, } diff --git a/tests/fixtures/figure-paragraph-spec.yaml b/tests/fixtures/figure-paragraph-spec.yaml new file mode 100644 index 0000000..d2f48ee --- /dev/null +++ b/tests/fixtures/figure-paragraph-spec.yaml @@ -0,0 +1,10 @@ +# A spec for figure-paragraph.md that matches the figure's caption with its +# `ignore` selector, so the image is dropped and the set is built without it. +# Everything else the spec covers, and the checks find nothing: the dropped +# image is the one fault of it. +ignore: [Header, "Para text~'Figure 1'"] +question: Para text~'^A ball' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/tests/fixtures/figure-paragraph.md b/tests/fixtures/figure-paragraph.md new file mode 100644 index 0000000..cbe2f1a --- /dev/null +++ b/tests/fixtures/figure-paragraph.md @@ -0,0 +1,18 @@ +# Tutorial Sheet 1 + +## Question 1 + +A ball is thrown straight up at $20\,\mathrm{m/s}$. + +![the ball](figures/ball.png) +Figure 1: the ball leaving the hand. + +(a) Find the greatest height it reaches. + +(b) Find its time of flight. + +## Solutions + +1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ + +1(b) $t = 2v/g = 4.08\,\mathrm{s}$ diff --git a/tests/test_package.py b/tests/test_package.py index 8ceb0a3..0bb56bb 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -151,6 +151,62 @@ def test_a_part_with_no_solution_is_a_warning_the_report_is_still_clean_for( assert report.warnings == [one.message for one in report.findings] +def beside_its_image(tmp_path, name: str) -> Path: + """A sheet copied into a folder of its own, beside the image it refers to.""" + folder = tmp_path / Path(name).stem + (folder / "figures").mkdir(parents=True) + shutil.copy(FIXTURES / name, folder / name) + shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") + return folder / name + + +def test_a_block_marked_ignore_whose_lines_hold_an_image_is_reported(tmp_path): + # The spec marks the figure's paragraph ignored by matching its caption. + # Nothing in2lambda checks says so, and the set built from the draft holds + # no image. + written = package.source_add(beside_its_image(tmp_path, "figure-paragraph.md")) + package.spec_run(written, FIXTURES / "figure-paragraph-spec.yaml") + + (dropped,) = package.ignored_images(written) + + assert (dropped.check, dropped.level) == ("coverage", package.ERROR) + assert (dropped.field, dropped.ranges) == ("b4", [[7, 8]]) + assert dropped.message == "b4 (lines 7-8) holds an image and is marked ignore." + + +def test_an_image_inside_a_question_is_nothing_to_report(tmp_path): + written = package.source_add(beside_its_image(tmp_path, "figure.md")) + package.spec_run(written, FIXTURES / "sheet-spec.yaml") + + assert package.ignored_images(written) == [] + + +def test_a_source_whose_bytes_are_not_text_has_no_ignored_image_to_read(tmp_path): + # A docx source is frozen as itself, so the file beside the draft is a zip. + # Reading it for a `![` is not what it is for, and must not end the run. + written = package.source_add(beside_its_image(tmp_path, "figure-paragraph.md")) + package.spec_run(written, FIXTURES / "figure-paragraph-spec.yaml") + package.frozen_source(written).write_bytes((FIXTURES / "ball.png").read_bytes()) + + assert package.ignored_images(written) == [] + + +def test_the_coverage_line_names_the_images_the_spec_dropped(tmp_path): + written = package.source_add(beside_its_image(tmp_path, "figure-paragraph.md")) + + coverage = package.spec_run(written, FIXTURES / "figure-paragraph-spec.yaml") + + assert str(coverage).endswith("; 1 image dropped: b4 (lines 7-8)") + + +def test_the_coverage_line_of_a_spec_that_dropped_none_is_unchanged(tmp_path): + written = package.source_add(beside_its_image(tmp_path, "figure.md")) + + coverage = package.spec_run(written, FIXTURES / "sheet-spec.yaml") + + assert str(coverage).endswith("4 ignored, none unassigned") + + def test_a_second_source_is_frozen_into_the_same_draft(tmp_path): folder = tmp_path / "sheets" folder.mkdir() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index eded260..22ac5f1 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -30,6 +30,7 @@ LONE_SPEC = (FIXTURES / "lone-spec.yaml").read_text() TEX_SPEC = (FIXTURES / "tex-sheet-spec.yaml").read_text() FAULTY_SPEC = (FIXTURES / "faulty-spec.yaml").read_text() +IGNORES_THE_FIGURE = (FIXTURES / "figure-paragraph-spec.yaml").read_text() # What a model would run over the faulty sheet: the merged block cut in two and # each half quoted, the solution the spec's selector missed given to the question @@ -195,6 +196,23 @@ def figures(tmp_path): return folder +@pytest.fixture +def figure_paragraph(tmp_path): + """A sheet whose figure is a paragraph of its own, under a spec that drops it. + + Mathpix writes a figure this way: the image line and then its caption, apart + from the question they illustrate. The saved spec matches that caption with + its `ignore` selector, so the run's first draft is one in2lambda's checks + have nothing to say about and the image is in no field all the same. + """ + folder = tmp_path / "figure-paragraph" + (folder / "figures").mkdir(parents=True) + shutil.copy(FIXTURES / "figure-paragraph.md", folder / "figure-paragraph.md") + shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") + (folder / SPEC_NAME).write_text(IGNORES_THE_FIGURE) + return folder + + @pytest.fixture def paired(tmp_path): """A sheet whose solutions are written as a file of their own beside it.""" @@ -431,6 +449,37 @@ def test_a_spec_in2lambda_will_not_run_stops_the_run_saying_why(sheets, tmp_path assert result.zip_path.exists() +def test_a_saved_spec_that_drops_an_image_is_written_again_and_the_run_goes_on( + figure_paragraph, tmp_path +): + # The checks find nothing in the draft the saved spec filled: the image it + # marked ignore is the only fault of it, and it is the one the run writes + # the spec again over. The rewrite keeps ignoring the figure, so the run + # reports the drop on its coverage line and builds the set without it. + backend = FakeBackend(IGNORES_THE_FIGURE) + + result = pipeline.run( + figure_paragraph / "figure-paragraph.md", + out_dir=tmp_path / "out", + settings=Settings(), + tries=1, + backend=backend, + ) + validated = [stage.message for stage in result.stages if stage.name == "validate"] + covered = [stage.message for stage in result.stages if stage.name == "coverage"] + + assert validated[0] == ( + "b4 (lines 7-8) holds an image and is marked ignore. — " + "writing the set's spec again" + ) + assert len(backend.calls) == 1 + assert "b4 (lines 7-8) holds an image" in backend.calls[0][1] + assert covered[-1].endswith("; 1 image dropped: b4 (lines 7-8)") + assert validated[-1] == "nothing to report" + assert result.reused is False + assert result.zip_path is not None and result.zip_path.exists() + + def test_a_rewrite_in2lambda_will_not_run_leaves_the_saved_spec_alone( sheets, tmp_path ): diff --git a/tests/test_review.py b/tests/test_review.py index 1a55ee8..7fc36d2 100644 --- a/tests/test_review.py +++ b/tests/test_review.py @@ -78,7 +78,7 @@ def test_the_record_goes_to_json_and_comes_back(tmp_path): saved = review( usage=Usage(input_tokens=120, output_tokens=40, seconds=1.5), tries=[ - SpecTry(0, Usage(), unassigned=2, errors=2), + SpecTry(0, Usage(), unassigned=2, errors=2, dropped=1), SpecTry(1, Usage(input_tokens=120, output_tokens=40), second=0, chosen=True), ], rounds=[ diff --git a/tests/test_spec.py b/tests/test_spec.py index 7a020bf..070bedd 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -1,25 +1,29 @@ """Layer 1: where a set's spec lives, and the one call that writes it.""" import json +import shutil from pathlib import Path import pytest from conftest import FakeBackend from in2lambda_agent.model import Usage -from in2lambda_agent.package import Coverage, Report +from in2lambda_agent.package import Coverage, Finding, Report from in2lambda_agent.spec import ( SPEC_NAME, BadSpec, Previous, Second, SpecTry, + iterate_spec, record_run, spec_path, write_spec, ) -SPEC = (Path(__file__).parent / "fixtures" / "sheet-spec.yaml").read_text() +FIXTURES = Path(__file__).parent / "fixtures" +SPEC = (FIXTURES / "sheet-spec.yaml").read_text() +IGNORES_THE_FIGURE = (FIXTURES / "figure-paragraph-spec.yaml").read_text() def test_the_sets_spec_is_beside_the_source(tmp_path): @@ -89,6 +93,73 @@ def test_a_revision_carries_the_last_spec_and_what_running_it_covered(): assert "over sheet-2.md, another document of this set, left b5, b6" in prompt +def test_a_revision_shows_the_images_the_last_spec_ignored(tmp_path): + backend = FakeBackend(SPEC) + previous = Previous( + text="question: Para\nlayout: PartsSepSol\n", + coverage=Coverage( + layout="PartsSepSol", + blocks=9, + fields={1: 5}, + ignored=4, + dropped=[ + Finding( + check="coverage", + level="error", + field="b4", + ranges=[[7, 8]], + message="b4 (lines 7-8) holds an image and is marked ignore.", + ) + ], + ), + report=Report(clean=True, errors=[]), + ) + + write_spec("b1 1 # Sheet", backend, previous) + + ((_, prompt),) = backend.calls + # Under the one heading as the checks' own errors: the next call answers an + # ignored figure the way it answers a block left in no field. + assert "The checks then found:\n\nb4 (lines 7-8) holds an image" in prompt + assert "fewer blocks unassigned, fewer images ignored" in prompt + + +def test_a_spec_that_ignores_a_figure_is_written_again_and_the_drop_reported(tmp_path): + # Every try marks the figure's paragraph ignored, so no try scores zero and + # the loop spends both its calls before keeping the first. + folder = tmp_path / "figure-paragraph" + (folder / "figures").mkdir(parents=True) + shutil.copy(FIXTURES / "figure-paragraph.md", folder / "figure-paragraph.md") + shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png") + backend = FakeBackend(IGNORES_THE_FIGURE, IGNORES_THE_FIGURE) + stages: list[tuple[str, str]] = [] + + _, coverage, report, tries = iterate_spec( + folder / "figure-paragraph.md", + folder / SPEC_NAME, + backend, + tries=2, + on_stage=lambda name, message: stages.append((name, message)), + ) + + assert len(backend.calls) == 2 + # The checks find nothing in either draft, so the dropped image is the whole + # of the score. + assert [(one.unassigned, one.errors, one.dropped) for one in tries] == [ + (0, 0, 1), + (0, 0, 1), + ] + assert [one.score for one in tries] == [1, 1] + # The revision names the image the first try dropped. + assert "b4 (lines 7-8) holds an image and is marked ignore." in backend.calls[1][1] + # The run goes on past the drop, and every coverage line names it. + assert report.clean is True + assert str(coverage).endswith("; 1 image dropped: b4 (lines 7-8)") + assert [message for name, message in stages if name == "coverage"] == [ + str(coverage) + ] * 3 + + def test_a_revision_of_a_spec_that_covered_the_set_says_so(): backend = FakeBackend(SPEC) previous = Previous( @@ -187,7 +258,7 @@ def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path) coverage=coverage, usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), tries=[ - SpecTry(number=0, unassigned=2, errors=2), + SpecTry(number=0, unassigned=2, errors=2, dropped=1), SpecTry( number=1, usage=Usage(input_tokens=900, output_tokens=80, seconds=2.5), @@ -214,6 +285,7 @@ def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path) "seconds": 0.0, "unassigned": 2, "errors": 2, + "dropped": 1, "second": None, "chosen": False, } @@ -222,6 +294,13 @@ def test_the_record_says_what_each_spec_the_run_wrote_covered_and_cost(tmp_path) assert (second["try"], second["seconds"], second["chosen"]) == (2, 2.0, True) +def test_a_try_is_scored_on_its_dropped_images_as_well(): + one = SpecTry(number=1, unassigned=1, errors=2, dropped=3, second=4) + + assert one.score == 10 + assert SpecTry(number=1).score == 0 + + def test_the_record_names_the_other_document_of_the_set(tmp_path): record = tmp_path / "runs.jsonl" coverage = Coverage(layout="PartsSepSol", blocks=14, fields={1: 10})