Copy a set's subfolders into the sweep's work directory

README.md

@@ -290,7 +290,9 @@ own, and is a row like any other document.
 
 The sweep never writes to the corpus. It copies each set's folder into `--work`
 (default `./.in2lambda-agent/corpus`), empties that copy first, and runs the documents
-there. It keeps three kinds of file in `--specs` (default `./corpus-specs`), in a tree
+there. The copy holds everything under the folder — the figures a sheet names among it —
+less what an earlier run left there: a spec, a draft, an `.in2lambda-agent` directory.
+It keeps three kinds of file in `--specs` (default `./corpus-specs`), in a tree
 mirroring the corpus: the set's spec, each document's log of the commands its fixing
 rounds ran, and the `in2lambda-agent-runs.jsonl` every run appends a line to. Set `A/B`
 keeps its spec at `corpus-specs/A/B/in2lambda-spec.yaml` and the log of `A/B/sheet.tex`

in2lambda_agent/corpus.py

@@ -190,13 +190,22 @@ def stage(
 ) -> Path:
     """Copies one set's folder into the work directory, and empties it first.
 
-    The set is the folder, and its sheets are the files in it: a subfolder
-    holding a document of its own is a set of its own and is left for its own
-    staging, while one holding none — figures, styles — comes along, since the
-    sheets refer to it. A tex file that is a drawing and not a sheet is no
-    document, so the folder of them is one of those that come along. What no run could read is left behind: the archives, and
-    the PDFs unless they are what is being run. They are most of what a corpus
-    weighs, and the copy is made again every sweep.
+    The set is the folder and everything under it: the sheets, and the figures,
+    styles and class files they name, wherever in the tree those sit. A
+    subfolder comes along whatever is in it, sheets of its own included, since
+    deciding that per folder is what dropped the figures PHYS40002's sheets
+    name. A subfolder that is a set in its own right is staged again under its
+    own name when the sweep reaches it, so the corpus root brings a second copy
+    of every set beneath it.
+
+    What no run could read is left behind: the archives, and the PDFs unless
+    they are what is being run. They are most of what a corpus weighs, and the
+    copy is made again every sweep. So is what an earlier run of the agent left
+    in the corpus, at whatever depth: a spec, a record, a draft, and the cache
+    directory. And so is the work directory itself when it sits under the set
+    being staged, as the default `--work` does under the corpus root: a copy of
+    it would hold a copy of itself, and so on until the path is too long for
+    the filesystem.
 
     A set staged twice is emptied first, so a sweep starts from nothing every
     time. That is the set's own folder and never the work directory, which holds
@@ -215,27 +224,30 @@ def stage(
     into = Path(work) / (ROOT_SET if relative == Path(".") else relative)
     if into.exists():
         shutil.rmtree(into)
-    into.mkdir(parents=True)
+    into.parent.mkdir(parents=True, exist_ok=True)
     pdfs = "pdf" in {one.lower().lstrip(".") for one in suffixes}
-    ignore = shutil.ignore_patterns(*(("*.zip",) if pdfs else ("*.zip", "*.pdf")))
-    named = [path.name for path in folder.iterdir()]
-    skipped = ignore(str(folder), named) | {SPEC_NAME, RECORD_NAME}
-    for path in folder.iterdir():
+    # Asked at every level of the tree, so what an earlier run left is left
+    # behind wherever in the set it sits.
+    by_name = shutil.ignore_patterns(
+        "*.zip",
+        *(() if pdfs else ("*.pdf",)),
+        SPEC_NAME,
+        RECORD_NAME,
         # A draft is named after the source it was frozen from, so there is one
         # per sheet rather than one per folder: the name is not known in advance
         # and the suffix is what says a file is one.
-        if path.name.endswith(package.DRAFT_SUFFIX):
-            continue
-        if path.is_dir():
-            # A folder of figures whose tex sources are drawings holds no
-            # document, so it comes along with the sheets that refer to it
-            # rather than being staged and run as a set of its own.
-            if not any(
-                is_document(one) for one in documents(path, suffixes=suffixes)
-            ):
-                shutil.copytree(path, into / path.name, ignore=ignore)
-        elif path.name not in skipped:
-            shutil.copy2(path, into / path.name)
+        "*" + package.DRAFT_SUFFIX,
+        pipeline.DEFAULT_CACHE_DIR.name,
+    )
+    target = Path(work).resolve()
+
+    def ignore(where: str, named: list[str]) -> set[str]:
+        """What is left behind in one folder: by name, and the work directory."""
+        return by_name(where, named) | {
+            one for one in named if (Path(where) / one).resolve() == target
+        }
+
+    shutil.copytree(folder, into, ignore=ignore)
     return into
 
 

tests/fixtures/tex-figure-spec.yaml

@@ -0,0 +1,10 @@
+# The spec for the sheet with a figure: the question is the paragraph that
+# names the image, its parts are the items of the list under it, and its
+# solutions are under the `Solutions` heading. A solution begins with its
+# label, `1(a)`, so no solution matches `question`.
+ignore:   Header
+question: Para text~'^[A-Z]'
+part:     ListItem
+solution: after Header text=Solutions, Para
+strip:    ['^\d+\([a-z]\) ']
+layout:   PartsSepSol

tests/fixtures/tex-figure.tex

@@ -0,0 +1,26 @@
+% Synthetic, in the shape of the corpus's sheets that carry a figure: a
+% question whose text names an image in a folder beside the sheet, and its
+% solutions under a heading. Nothing here is copied from ExampleContents.
+\documentclass[12pt]{article}
+\usepackage{amsmath}
+\usepackage{graphicx}
+
+\begin{document}
+
+\section*{Problem Sheet 1: Statics}
+
+The beam below carries a load $W$ at its midpoint.
+\includegraphics[width=0.2\textwidth]{figures/ball.png}
+
+\begin{enumerate}
+\item Find the reaction at each support.
+\item Find the bending moment at the midpoint.
+\end{enumerate}
+
+\section*{Solutions}
+
+1(a) $W/2$ at each support.
+
+1(b) $M = WL/4$
+
+\end{document}

tests/test_corpus.py

@@ -15,6 +15,7 @@
 from in2lambda_agent.spec import SPEC_NAME
 
 FIXTURES = Path(__file__).parent / "fixtures"
+TEX_FIGURE_SPEC = (FIXTURES / "tex-figure-spec.yaml").read_text()
 
 
 def make_set(root, folder, names):
@@ -481,14 +482,43 @@ def test_a_staged_set_leaves_behind_what_no_run_reads(root, tmp_path):
     (root / "sheets" / "figures").mkdir()
     (root / "sheets" / "figures" / "plot.png").write_bytes(b"PNG")
     (root / "sheets" / "figures" / "page.pdf").write_bytes(b"%PDF-1.4")
+    # What a run in the corpus itself would have left: a draft beside the sheet
+    # it was frozen from, and a cache directory — here under the figures, to
+    # show that the depth it sits at makes no difference.
+    (root / "sheets" / ("sheet" + package.DRAFT_SUFFIX)).write_text("{}")
+    cache = root / "sheets" / "figures" / pipeline.DEFAULT_CACHE_DIR.name
+    cache.mkdir()
+    (cache / "sheet.md").write_text("# an OCR conversion from a run before")
 
     staged = corpus.stage(root, root / "sheets", tmp_path / "work", ("md",))
 
     # The figures come along, since the sheets refer to them; the PDFs do not,
-    # in the folder itself or under it.
+    # in the folder itself or under it, and neither does anything an earlier
+    # run wrote.
     assert contents(staged) == ["figures/plot.png", "sheet-2.md", "sheet.md"]
 
 
+def test_a_set_with_figures_in_a_subfolder_sweeps_to_built(tmp_path):
+    # What PHYS40002's sheets do: name an image in a folder beside them. The
+    # folder has to come along, or the build faults on a file it cannot find.
+    # The caption among the figures is a document, so the folder is a set of
+    # its own when the sweep reaches it — and that is no reason to leave it
+    # behind when the sheet that names its image is staged.
+    made = tmp_path / "corpus"
+    folder = make_set(made, "beams", ["tex-figure.tex"])
+    (folder / "figures").mkdir()
+    shutil.copy(FIXTURES / "ball.png", folder / "figures" / "ball.png")
+    (folder / "figures" / "caption.md").write_text("# Figure 1\n\nA loaded beam.\n")
+
+    rows = sweep(made, tmp_path, backend=FakeBackend(TEX_FIGURE_SPEC, TEX_FIGURE_SPEC))
+
+    (row,) = [one for one in rows if one.source == "beams/tex-figure.tex"]
+    # The build compiles the set as the PDF generator does, so it is `built`
+    # only because the image was there to load.
+    assert row.outcome == "built", row.reason
+    assert "figures/ball.png" in contents(tmp_path / "work" / "beams")
+
+
 def test_the_corpus_root_is_a_set_of_its_own_and_wipes_nothing_but_itself(
     root, tmp_path
 ):
@@ -497,6 +527,8 @@ def test_the_corpus_root_is_a_set_of_its_own_and_wipes_nothing_but_itself(
     # and take the sets already staged beside it — and anything else a
     # user-named --work holds — with it when it is emptied.
     shutil.copy(FIXTURES / "sheet.md", root / "loose.md")
+    (root / "figures").mkdir()
+    (root / "figures" / "plot.png").write_bytes(b"PNG")
     work = tmp_path / "work"
     already = corpus.stage(root, root / "sheets", work, ("md", "tex"))
     (work / "not-the-sweep's.txt").write_text("a user's own --work")
@@ -505,11 +537,39 @@ def test_the_corpus_root_is_a_set_of_its_own_and_wipes_nothing_but_itself(
     corpus.stage(root, root, work, ("md", "tex"))
 
     assert staged.parent == work and staged.name == corpus.ROOT_SET
-    assert contents(staged) == ["loose.md"]
+    # Everything under the root, the sets beneath it included: they are staged
+    # again under their own names when the sweep reaches them, and a second
+    # copy of them here costs less than deciding per folder what to bring.
+    assert contents(staged) == [
+        "figures/plot.png",
+        "loose.md",
+        "sheets/sheet-2.md",
+        "sheets/sheet.md",
+        "tex/tex-sheet-2.tex",
+        "tex/tex-sheet.tex",
+    ]
     assert contents(already) == ["sheet-2.md", "sheet.md"]
     assert (work / "not-the-sweep's.txt").exists()
 
 
+def test_a_work_directory_inside_the_corpus_is_not_copied_into_itself(root, tmp_path):
+    # The default --work sits under the corpus root, which is a set of its own:
+    # its copy would hold the work directory, which holds that copy, and so on
+    # until the path is too long for the filesystem. So the work directory is
+    # left behind by where it is, whatever the user named it.
+    work = root / "work"
+    corpus.stage(root, root / "sheets", work, ("md",))
+
+    staged = corpus.stage(root, root, work, ("md",))
+
+    assert contents(staged) == [
+        "sheets/sheet-2.md",
+        "sheets/sheet.md",
+        "tex/tex-sheet-2.tex",
+        "tex/tex-sheet.tex",
+    ]
+
+
 TIKZ = """\\begin{tikzpicture}
   \\draw[->] (0,0) -- (4,0) node[right] {$x$};
 \\end{tikzpicture}