From bd44c7cbc17c088216983854a5f76095e79ec8e5 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 02:19:31 +0100 Subject: [PATCH] implement: Convert a solutions file on its own when no questions file is beside it (t30) --- README.md | 7 ++-- docs/how-it-works.md | 19 +++++++++-- in2lambda_agent/cli.py | 5 +-- in2lambda_agent/corpus.py | 19 ++++------- in2lambda_agent/pair.py | 26 +++++--------- in2lambda_agent/pipeline.py | 14 ++++++-- in2lambda_agent/spec.py | 14 ++++++++ tests/fixtures/lone-spec.yaml | 8 +++++ tests/fixtures/lone_solutions.md | 11 ++++++ tests/test_corpus.py | 22 ++++++------ tests/test_docs.py | 2 +- tests/test_pair.py | 16 ++++----- tests/test_pipeline.py | 58 +++++++++++++++++--------------- tests/test_spec.py | 12 +++++++ 14 files changed, 142 insertions(+), 91 deletions(-) create mode 100644 tests/fixtures/lone-spec.yaml create mode 100644 tests/fixtures/lone_solutions.md diff --git a/README.md b/README.md index 2f30217..e377485 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ the one beside `SOURCE` whose name is the sheet's with `_solutions`, `-solutions and `Worksheet_1_solutions.pdf`. Both are frozen into the one draft, the questions first and the solutions second, and the run is named after the questions file whether you name that file or the solutions one. A solutions file with no questions file -beside it stops the run, which exits 1 saying `solutions without questions`. +beside it is converted on its own, and a `pair` line names the questions file the +agent looked for. The marker above each group of solutions is that run's question. The run writes a spec — the YAML selectors naming which blocks of the source are questions, parts and solutions — in one model call, and saves it as @@ -261,8 +262,8 @@ pdf` runs the PDFs too, which needs Mathpix credentials and one call per PDF. Ev is review mode `none`. A sheet and the solutions file beside it are one run and one row, named after the -questions file. A solutions file with no questions file beside it is a `skipped` row -with the reason `solutions without questions`. +questions file. A solutions file with no questions file beside it is converted on its +own, and is a row like any other document. The sweep never writes to the corpus. It copies each set's folder into `--work` (default `./.in2lambda-agent/corpus`), empties that copy first, and runs the documents diff --git a/docs/how-it-works.md b/docs/how-it-works.md index ad22e5d..69034d2 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -26,12 +26,14 @@ review not asked for (mode none) build /home/me/sheets/out/set.zip ``` -There are nine stage names: `ocr`, `freeze`, `spec`, `coverage`, `validate`, `fix`, -`render`, `review` and `build`. A run prints `validate` once per check and `fix` once -per fixing round, so those two names repeat. +There are ten stage names: `pair`, `ocr`, `freeze`, `spec`, `coverage`, `validate`, +`fix`, `render`, `review` and `build`. A run prints `pair` only for a solutions file +with no questions file beside it. A run prints `validate` once per check and `fix` +once per fixing round, so those two names repeat. | Stage | in2lambda function | What the stage writes | | --- | --- | --- | +| `pair` | none: the agent reads the file names | nothing | | `ocr` | none: Mathpix converts the PDF | `CACHE/HASH/source.md` and `CACHE/HASH/media/` | | `freeze` | `in2lambda.source.add` | `SOURCE.draft.json`, beside the frozen source | | `spec` | `in2lambda.source.show`, for the model's prompt | `in2lambda-spec.yaml`, beside `SOURCE` or at `--spec` | @@ -49,6 +51,17 @@ A run also appends one line to `in2lambda-agent-runs.jsonl`, beside the spec. Th appends that line last, whether it wrote a zip or ended with a report. A run that stops for a review appends no line: the approval that completes the review appends it. +### `pair` + +A sheet and the solutions file beside it are one run, named after the questions file. +The stage prints one message, and only where the source is a solutions file that the +folder holds no questions file for: + +* `no questions file named Tutorial_2.pdf beside Tutorial_2_Solutions.pdf; converting + the solutions alone` — the run converts the solutions file as a document of its own. + The spec prompt then says that the document holds solutions and no questions, so the + model writes a `question` selector for the marker above each group of solutions. + ### `ocr` The stage converts a PDF to markdown through Mathpix. A source of any other suffix is diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index 7664e72..8ddfc4a 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -11,7 +11,6 @@ from in2lambda_agent.model import ModelUnavailable, choose_backend from in2lambda_agent.ocr import ocr_pdf from in2lambda_agent.package import CommandRefused, SpecRejected -from in2lambda_agent.pair import SolutionsWithoutQuestions from in2lambda_agent.review import ReviewError from in2lambda_agent.settings import load_settings from in2lambda_agent.spec import BadSpec @@ -359,12 +358,10 @@ def main(argv: Optional[Sequence[str]] = None) -> int: SpecRejected, ReviewError, CommandRefused, - SolutionsWithoutQuestions, ) as error: # Missing credentials among them: the message names the variables, or # the login to run, or what a spec says that a spec cannot say, or the - # question a review command names that is not under review, or the - # questions file a solutions file was run without. + # question a review command names that is not under review. print(f"in2lambda-agent: {error}", file=sys.stderr) return 1 diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 494d19c..d5d0bb0 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -427,19 +427,12 @@ def sweep( continue # A solutions document is frozen into the run of the questions document # it answers, so the pair is one row, which is the questions document's. - # One with no questions document beside it has no questions to attach - # its solutions to, and is a row of its own saying so. - if pair.questions_stem(document) is not None: - if pair.questions_beside(document) is not None: - continue - row = Row( - source=relative.as_posix(), - set=relative.parent.as_posix(), - outcome="skipped", - reason="solutions without questions", - ) - print(f"{row.outcome:<20} {row.source}") - rows.append(row) + # One with no questions document beside it is converted on its own, and + # is a row like any other document. + if ( + pair.questions_stem(document) is not None + and pair.questions_beside(document) is not None + ): continue # Staging is per set and the row is per document, so the guard is here # rather than in `run_one`: what a copy raises — an unreadable folder, a diff --git a/in2lambda_agent/pair.py b/in2lambda_agent/pair.py index 1e5c25a..a52be82 100644 --- a/in2lambda_agent/pair.py +++ b/in2lambda_agent/pair.py @@ -1,10 +1,10 @@ """Which document holds the solutions to which, going by the file names. A folder of worksheets often writes the questions and the solutions as separate -documents: `Worksheet_1.pdf` beside `Worksheet_1_solutions.pdf`. A solutions -document alone holds no questions for its solutions to answer, so the run +documents: `Worksheet_1.pdf` beside `Worksheet_1_solutions.pdf`. The run freezes the two into one draft, the questions first and the solutions second, -and is named after the questions document. +and is named after the questions document. A solutions document with no +questions document beside it is converted on its own. The pairing is by name. A solutions document is one whose stem ends in `solutions` after a space, an underscore or a hyphen, in any case. Its questions @@ -24,10 +24,6 @@ """ -class SolutionsWithoutQuestions(ValueError): - """A solutions document has no questions document beside it.""" - - def questions_stem(document: Path) -> Optional[str]: """The stem of the questions document a solutions document answers. @@ -110,20 +106,14 @@ def of(source: Path) -> tuple[Path, Optional[Path]]: Returns: The questions document, and the solutions document to freeze after it, - or None where the folder holds no solutions document for it. - - Raises: - SolutionsWithoutQuestions: `source` is a solutions document and the - folder holds no questions document for it. + or None where the folder holds no solutions document for it. A + solutions document with no questions document beside it is returned as + the first of the two, so the run converts it on its own. """ source = Path(source) - stem = questions_stem(source) - if stem is None: + if questions_stem(source) is None: return source, solutions_beside(source) questions = questions_beside(source) if questions is None: - raise SolutionsWithoutQuestions( - f"solutions without questions: nothing named {stem}{source.suffix} " - f"beside {source.name}" - ) + return source, None return questions, source diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index fe721d1..8b92a46 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -160,8 +160,6 @@ def run( BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. SourceError: If in2lambda cannot freeze or check the source. - SolutionsWithoutQuestions: If `source` is a solutions file and no - questions file is beside it. """ # A relative --out means the directory the user ran from, whatever in2lambda # does with the working directory along the way. @@ -174,11 +172,22 @@ def run( # file's from here on, whichever of the two the user named. source, solutions = pair.of(source) + # A solutions file with no questions file beside it is converted on its + # own. Its questions are the markers written above its solutions, which the + # spec prompt says so that the model writes `question` selectors for them. + alone = pair.questions_stem(source) if solutions is None else None + # The set is the folder the user's file is in, so this is settled before # OCR moves a PDF's markdown off into the cache. saved = spec_path(source, spec) result = RunResult(on_stage=on_stage) + if alone is not None: + result.add_stage( + "pair", + f"no questions file named {alone}{source.suffix} beside " + f"{source.name}; converting the solutions alone", + ) # The rest of the pipeline reads markdown, so a PDF becomes markdown first. frozen, _, message = _markdown( @@ -236,6 +245,7 @@ def run( backend, report if report.errors else None, sources=2 if frozen_solutions is not None else 1, + solutions_only=alone is not None, ) if saved.is_file(): replaced = saved.read_text(encoding="utf-8") diff --git a/in2lambda_agent/spec.py b/in2lambda_agent/spec.py index 2269fd2..d76f9fc 100644 --- a/in2lambda_agent/spec.py +++ b/in2lambda_agent/spec.py @@ -138,6 +138,7 @@ def write_spec( backend: Backend, report: Optional[Report] = None, sources: int = 1, + solutions_only: bool = False, ) -> tuple[str, Reply]: """Writes a spec for a source, in one model call with no tools. @@ -148,6 +149,10 @@ def write_spec( where this is the rewrite that follows a dirty validate. sources: How many documents the draft holds: 2 where the solutions are a file of their own, which the prompt then says before the source. + solutions_only: Whether the one document is a file of solutions with no + questions file beside it, which the prompt then says before the + source, because the questions of such a document are the markers + written above its solutions. Returns: The spec, and the reply it came in. @@ -163,6 +168,15 @@ def write_spec( "are `b1` onwards, and its solutions file, whose blocks are `2/b1` " "onwards. Every solution is in the second.\n\n" ) + elif solutions_only: + prompt = ( + "This document holds solutions and no questions, and there is no " + "second source. The marker written above each group of solutions — " + "the `Q2.` or the `## Question 2` — is the question here, and its " + "text is that question's text. Write `question` to match every " + "marker, and `solution` to match the worked solutions under it. " + "`layout` describes this file.\n\n" + ) prompt += f"Here is the source, one line each with its block id:\n\n{shown}\n" if report is not None: prompt += ( diff --git a/tests/fixtures/lone-spec.yaml b/tests/fixtures/lone-spec.yaml new file mode 100644 index 0000000..ab95b21 --- /dev/null +++ b/tests/fixtures/lone-spec.yaml @@ -0,0 +1,8 @@ +# The spec the fake backend answers with for a file of solutions that has no +# questions file beside it. The marker above each solution is the question, so +# the `question` selector matches the markers and the `solution` selector the +# paragraphs under them. +ignore: Header +question: Para text~'^Q\d' +solution: after Header text=Solutions, Para +layout: PartsOneSol diff --git a/tests/fixtures/lone_solutions.md b/tests/fixtures/lone_solutions.md new file mode 100644 index 0000000..00757cf --- /dev/null +++ b/tests/fixtures/lone_solutions.md @@ -0,0 +1,11 @@ +# Tutorial Sheet 5: answers + +## Solutions + +Q1. + +$\omega = v / r$ + +Q2. + +$T = 2\pi\sqrt{m/k}$ diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 38d2f46..66e9860 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -7,7 +7,7 @@ import pytest from conftest import FakeBackend -from test_pipeline import FAULTY_SPEC, FIXES, PAIRED_SPEC, SPEC, TEX_SPEC +from test_pipeline import FAULTY_SPEC, FIXES, LONE_SPEC, PAIRED_SPEC, SPEC, TEX_SPEC from in2lambda_agent import corpus, pipeline from in2lambda_agent.settings import Settings @@ -498,21 +498,19 @@ def test_a_sheet_and_its_solutions_file_are_one_row(tmp_path): ] -def test_solutions_with_no_questions_beside_them_are_skipped(tmp_path): +def test_solutions_with_no_questions_beside_them_are_converted_alone(tmp_path): root = tmp_path / "corpus" folder = root / "worksheets" folder.mkdir(parents=True) - shutil.copy( - FIXTURES / "paired_solutions.md", folder / "Tutorial_2_Solutions.md" - ) - backend = FakeBackend(PAIRED_SPEC) + shutil.copy(FIXTURES / "lone_solutions.md", folder / "Tutorial_2_Solutions.md") + backend = FakeBackend(LONE_SPEC) rows = sweep(root, tmp_path, backend=backend) - # `skipped` is an outcome the command exits 0 on, as a drawing's row is. - assert [(row.source, row.outcome, row.reason) for row in rows] == [ - ("worksheets/Tutorial_2_Solutions.md", "skipped", "solutions without questions") + # A document of solutions is a set of its own, so the sweep runs it and + # records the row it built. + assert [(row.source, row.set, row.outcome) for row in rows] == [ + ("worksheets/Tutorial_2_Solutions.md", "worksheets", "built") ] - # Nothing was frozen or called on its account. - assert len(backend.calls) == 0 - assert not (tmp_path / "work" / "worksheets").exists() + assert len(backend.calls) == 1 + assert (tmp_path / "work" / "worksheets" / "Tutorial_2_Solutions.md").is_file() diff --git a/tests/test_docs.py b/tests/test_docs.py index 35b43e0..cc980fd 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -57,6 +57,6 @@ def test_readme_names_every_option(): def test_how_it_works_names_every_stage(): names = _stage_names() - assert len(names) == 9 + assert len(names) == 10 missing = [one for one in sorted(names) if f"`{one}`" not in HOW_IT_WORKS] assert not missing diff --git a/tests/test_pair.py b/tests/test_pair.py index a7ed312..0a88053 100644 --- a/tests/test_pair.py +++ b/tests/test_pair.py @@ -71,16 +71,16 @@ def test_a_folder_that_is_not_there_holds_no_companion(tmp_path): assert pair.questions_beside(absent / "Worksheet_1_solutions.pdf") is None assert pair.solutions_beside(absent / "Worksheet_1.pdf") is None assert pair.of(absent / "Worksheet_1.pdf") == (absent / "Worksheet_1.pdf", None) - with pytest.raises(pair.SolutionsWithoutQuestions): - pair.of(absent / "Worksheet_1_solutions.pdf") + assert pair.of(absent / "Worksheet_1_solutions.pdf") == ( + absent / "Worksheet_1_solutions.pdf", + None, + ) -def test_solutions_with_no_questions_say_what_is_missing(tmp_path): +def test_solutions_with_no_questions_run_alone(tmp_path): + # The markers above the solutions are this document's questions, so the + # document converts with no second file. solutions = tmp_path / "Tutorial_2_Solutions.pdf" solutions.write_bytes(b"%PDF") - with pytest.raises( - pair.SolutionsWithoutQuestions, - match="solutions without questions: nothing named Tutorial_2.pdf", - ): - pair.of(solutions) + assert pair.of(solutions) == (solutions, None) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index dc8fc32..e849d53 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -12,7 +12,7 @@ from conftest import PNG, FakeBackend, FakeMathpix from in2lambda.validation.pdf import missing_tools -from in2lambda_agent import package, pair, pipeline +from in2lambda_agent import package, pipeline from in2lambda_agent.cli import main from in2lambda_agent.model import ModelUnavailable from in2lambda_agent.package import SpecRejected @@ -24,6 +24,7 @@ SOURCE = FIXTURES / "sheet.md" SPEC = (FIXTURES / "sheet-spec.yaml").read_text() PAIRED_SPEC = (FIXTURES / "paired-spec.yaml").read_text() +LONE_SPEC = (FIXTURES / "lone-spec.yaml").read_text() TEX_SPEC = (FIXTURES / "tex-sheet-spec.yaml").read_text() FAULTY_SPEC = (FIXTURES / "faulty-spec.yaml").read_text() @@ -1720,36 +1721,39 @@ def test_naming_the_solutions_file_runs_the_questions_file(paired, tmp_path): assert result.zip_path.exists() -def test_solutions_with_no_questions_beside_them_stop_the_run(tmp_path): +def test_solutions_with_no_questions_beside_them_are_converted_alone(tmp_path): folder = tmp_path / "lone" folder.mkdir() - shutil.copy(FIXTURES / "paired_solutions.md", folder / "paired_solutions.md") + shutil.copy(FIXTURES / "lone_solutions.md", folder / "lone_solutions.md") + backend = FakeBackend(LONE_SPEC) - with pytest.raises( - pair.SolutionsWithoutQuestions, - match="solutions without questions: nothing named paired.md", - ): - pipeline.run( - folder / "paired_solutions.md", - out_dir=tmp_path / "out", - settings=Settings(), - backend=FakeBackend(PAIRED_SPEC), - ) - - -def test_a_lone_solutions_file_exits_one_saying_so(tmp_path, capsys): - folder = tmp_path / "lone" - folder.mkdir() - shutil.copy(FIXTURES / "paired_solutions.md", folder / "paired_solutions.md") - - code = main( - ["run", str(folder / "paired_solutions.md"), "--out", str(tmp_path / "out")] + result = pipeline.run( + folder / "lone_solutions.md", + out_dir=tmp_path / "out", + settings=Settings(), + backend=backend, ) - printed = capsys.readouterr() - - assert code == 1 - assert "solutions without questions" in printed.err - assert printed.out == "" + stages = {stage.name: stage.message for stage in result.stages} + ((_, prompt),) = backend.calls + + assert stages["pair"] == ( + "no questions file named lone.md beside lone_solutions.md; " + "converting the solutions alone" + ) + # One source, so the freeze line names the draft and nothing else. + assert stages["freeze"] == str(drafted(folder, "lone_solutions.md")) + assert "This document holds solutions and no questions" in prompt + assert result.draft == drafted(folder, "lone_solutions.md") + # The run is the solutions file's, so the record names it. + (line,) = (folder / RECORD_NAME).read_text().splitlines() + assert json.loads(line)["source"] == str(folder / "lone_solutions.md") + # The marker above each solution is a question, and the paragraph under it + # is that question's solution. + zip_file = zipfile.ZipFile(result.zip_path) + assert [ + json.loads(zip_file.read(name))["parts"][0]["workedSolution"]["content"] + for name in ("question_000_Question_1.json", "question_001_Question_2.json") + ] == ["$\\omega = v / r$", "$T = 2\\pi\\sqrt{m/k}$"] def test_a_pair_of_pdfs_is_converted_and_frozen_into_one_draft(tmp_path): diff --git a/tests/test_spec.py b/tests/test_spec.py index 381b6b0..87065f5 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -52,6 +52,18 @@ def test_the_prompt_carries_the_numbered_source_as_it_was_shown(): assert reply.usage.output_tokens > 0 +def test_a_file_of_solutions_alone_is_said_to_hold_no_questions(): + backend = FakeBackend(SPEC, SPEC) + + write_spec("b1 1 ## Solutions", backend, solutions_only=True) + write_spec("b1 1 ## Question 1", backend) + + lone, paired = (prompt for _, prompt in backend.calls) + assert "This document holds solutions and no questions" in lone + assert "is the question here" in lone + assert "holds solutions and no questions" not in paired + + def test_a_rewrite_carries_what_the_checks_found(): backend = FakeBackend(SPEC) report = Report(clean=False, errors=["b4 (lines 7-7) is in no field."])