From 93610dd036a4beb11d70f577904b9ec42ba811a0 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 00:41:11 +0100 Subject: [PATCH 1/6] implement: Gate every merge on a replay sweep over the corpus (t21) --- .github/branch-protection.json | 9 + .github/workflows/gate.yml | 100 ++++++++ .gitignore | 6 + README.md | 63 ++++- ci-corpus/docx/sheet.docx | Bin 0 -> 11101 bytes ci-corpus/pdf/.gitkeep | 0 ci-corpus/tex/figures/ball.png | Bin 0 -> 70 bytes ci-corpus/tex/sheet-1.tex | 46 ++++ ci-corpus/tex/sheet-2.tex | 28 +++ ci-corpus/tex/sheet-3.tex | 50 ++++ ci-corpus/tex/solutions-2.tex | 27 +++ .../in2lambda-spec.yaml | 6 + .../in2lambda-spec.yaml | 5 + corpus-specs/UCL_MechEng/in2lambda-spec.yaml | 5 + corpus-specs/baseline.json | 26 +++ .../ci-corpus/docx/in2lambda-spec.yaml | 6 + .../ci-corpus/pdf/in2lambda-spec.yaml | 5 + .../ci-corpus/tex/in2lambda-spec.yaml | 4 + in2lambda_agent/cli.py | 59 ++++- in2lambda_agent/corpus.py | 7 + in2lambda_agent/gate.py | 219 ++++++++++++++++++ in2lambda_agent/ocr.py | 27 ++- in2lambda_agent/pipeline.py | 14 +- tests/test_cli.py | 103 +++++++- tests/test_corpus.py | 16 ++ tests/test_gate.py | 198 ++++++++++++++++ tests/test_ocr.py | 15 +- tests/test_pipeline.py | 21 ++ 28 files changed, 1054 insertions(+), 11 deletions(-) create mode 100644 .github/branch-protection.json create mode 100644 .github/workflows/gate.yml create mode 100644 ci-corpus/docx/sheet.docx create mode 100644 ci-corpus/pdf/.gitkeep create mode 100644 ci-corpus/tex/figures/ball.png create mode 100644 ci-corpus/tex/sheet-1.tex create mode 100644 ci-corpus/tex/sheet-2.tex create mode 100644 ci-corpus/tex/sheet-3.tex create mode 100644 ci-corpus/tex/solutions-2.tex create mode 100644 corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml create mode 100644 corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml create mode 100644 corpus-specs/UCL_MechEng/in2lambda-spec.yaml create mode 100644 corpus-specs/baseline.json create mode 100644 corpus-specs/ci-corpus/docx/in2lambda-spec.yaml create mode 100644 corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml create mode 100644 corpus-specs/ci-corpus/tex/in2lambda-spec.yaml create mode 100644 in2lambda_agent/gate.py create mode 100644 tests/test_gate.py diff --git a/.github/branch-protection.json b/.github/branch-protection.json new file mode 100644 index 0000000..2d1362c --- /dev/null +++ b/.github/branch-protection.json @@ -0,0 +1,9 @@ +{ + "required_status_checks": { + "strict": false, + "contexts": ["gate"] + }, + "enforce_admins": false, + "required_pull_request_reviews": null, + "restrictions": null +} diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml new file mode 100644 index 0000000..9a5e845 --- /dev/null +++ b/.github/workflows/gate.yml @@ -0,0 +1,100 @@ +# The merge gate. Every push to main and every pull request runs the tests and +# then replays the folders corpus-specs/baseline.json names, which today are +# the three of ci-corpus, the committed corpus. The job is a required status +# check on main, so a branch that breaks a build cannot be merged. +name: gate + +on: + push: + branches: [main] + pull_request: + +jobs: + gate: + name: gate + # Pinned rather than ubuntu-latest: the runner's TeX Live decides what + # xelatex writes, and the PDF's bytes are the OCR cache's key. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + # The Dockerfile's package list, which is the one record of what the PDF + # generator's template.latex loads. + - name: Install pandoc, poppler and TeX Live + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + pandoc \ + poppler-utils \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-science \ + texlive-lang-chinese \ + texlive-lang-arabic \ + texlive-bibtex-extra \ + texlive-plain-generic \ + texlive-fonts-recommended \ + lmodern \ + fonts-noto-core \ + fonts-noto-cjk + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install poetry + run: pipx install poetry + + - name: Install the package + run: poetry install --with dev + + # SOURCE_DATE_EPOCH fixes the timestamp xelatex writes into the PDF, so + # the same tex source compiles to the same bytes and the OCR cache key + # does not change between runs. + - name: Compile the CI corpus PDF + run: | + cd ci-corpus/tex + SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ + xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex + rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log + + # Keyed by the PDF's bytes, which is also how ocr_pdf names its entry + # inside the directory. A restored entry for another PDF is unused. + - name: Restore the OCR cache + uses: actions/cache@v4 + with: + path: ~/.cache/in2lambda-agent + key: ocr-${{ hashFiles('ci-corpus/pdf/*.pdf') }} + restore-keys: ocr- + + - name: Tests + run: poetry run pytest -q + + - name: Gate + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: poetry run in2lambda-agent gate corpus-specs/baseline.json + + # What this run would record, so that a baseline change is committed from + # what CI saw. Written whether or not the gate passed. + - name: Record what this run did + if: always() + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: | + cp corpus-specs/baseline.json "${RUNNER_TEMP}/baseline.json" + poetry run in2lambda-agent gate --record "${RUNNER_TEMP}/baseline.json" + + - name: Upload it + if: always() + uses: actions/upload-artifact@v4 + with: + name: recorded-baseline + path: ${{ runner.temp }}/baseline.json diff --git a/.gitignore b/.gitignore index 45e2c81..b7e303d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,9 @@ __pycache__/ dist/ # in2lambda's KaTeX converter writes this into the working directory on import. log +# The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex +# in the workflow, so the repository holds the tex source and not the output. +ci-corpus/pdf/*.pdf +# Every run appends a line to the record beside the spec it read. The spec is +# what the gate replays; the record is one run's own and is not committed. +corpus-specs/**/in2lambda-agent-runs.jsonl diff --git a/README.md b/README.md index a3136a4..311908c 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ poetry run in2lambda-agent corpus ExampleContents --suffix tex --suffix md In full: ```sh -poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] +poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] [--cache DIR] ``` `ROOT` is the corpus directory and each `PATH` a folder under it to run, defaulting to @@ -209,6 +209,67 @@ how far the spec got alone — which is why a `built` row can still report block unassigned. `rejections` is always 0 while `--review` acts on nothing, and is the column a later review mode fills. +`--cache` (default `./.in2lambda-agent`) is where the OCR of each PDF is kept. A sweep +pointed at a cache that an earlier run filled makes no Mathpix call, and needs no +Mathpix credentials. + +## Gate + +Nothing merges without a replay over real documents. `gate` reruns the saved specs +over the folders a baseline file names, and compares how many documents each folder +built with the count the baseline records: + +```sh +poetry run in2lambda-agent gate corpus-specs/baseline.json [--record] [--cache DIR] [--work DIR] +``` + +Every run is `corpus --replay`, so no model call is made. The command prints one line +per folder, saying what the folder did beside what the baseline records, and exits 1 +when a folder builds fewer documents than the recorded count: + +``` +work /tmp/in2lambda-agent-gate-3f1a +ci-corpus/tex built 0 faulted 4 build refused 0 skipped 0 (baseline built 4) +ci-corpus/docx built 1 faulted 0 build refused 0 skipped 0 (baseline built 1) +ci-corpus/pdf built 1 faulted 0 build refused 0 skipped 0 (baseline built 1) +``` + +Run the command from the repository root: `specs`, and a folder's `root` where it is +relative, are read from there. + +`corpus-specs/baseline.json` is committed. Each folder's `root` and `suffixes` are +written by hand; `built` is what `--record` writes. A folder the file records no +`built` for passes on any count, and its line reads `(not recorded)`, which is how a +folder joins the gate before its specs replay to a build worth defending. A change to +a recorded count belongs in a pull request that says why the count changed. + +Today the file names the three folders of `ci-corpus`, the committed corpus: three tex +sheets, one docx, and one PDF that the workflow compiles from `ci-corpus/tex/sheet-1.tex` +with xelatex. The three folders of `ExampleContents`, the private corpus, have their +specs under `corpus-specs/` and are not in the file yet: every document there replays +to `faulted`, because pandoc's line wrapping is reported as a math delimiter error and +each one needs a fixing round that a replay does not run. + +`--cache` defaults to `~/.cache/in2lambda-agent`, outside any worktree, so that a PDF +converted on one branch is not converted again on the next. `--work` defaults to a new +directory under the system temp directory, which the gate does not delete: the drafts +of a folder that failed are read there. Neither directory is inside the repository, so +`git status` after a gate run reports nothing new. + +`.github/workflows/gate.yml` runs pytest and then the gate on every push to `main` and +every pull request. Mathpix reads the PDF once and the workflow stores the markdown in +the Actions cache under the PDF's hash, so a later run makes no call. The job needs two +repository secrets, `MATHPIX_APP_ID` and `MATHPIX_API_KEY`. A pull request from a fork +is given neither, and its `ci-corpus/pdf` folder faults; the tex and docx folders still +run. + +The `gate` job is a required status check on `main`: + +```sh +gh api -X PUT repos/{owner}/{repo}/branches/main/protection \ + --input .github/branch-protection.json +``` + ## Docker The image carries pandoc, a TeX Live with xelatex able to run the PDF generator's diff --git a/ci-corpus/docx/sheet.docx b/ci-corpus/docx/sheet.docx new file mode 100644 index 0000000000000000000000000000000000000000..137c8d0840d197d2a6afd26ca399d33777145ea1 GIT binary patch literal 11101 zcmZ{KWmsI<5-rfUySuwP!65|K;O_439^BnExO;GS_uxSj+#Mb>cka!E_g;T}y1)Lh z>g=vvwNI^8B_|00MFj$Zgaq2FVAX;&{WcK*3dfaBa?k+h7%r8h&*C;Mb}+ z?7^89VwJdN+Fzx2PE*22+~gd}uhq$(YLafXCs2p0LFw>Nb0W)T*eOOWAd#i4AL4#6 z(*RrovX}}M+;;oCDv8o0Wf^7|6)mSNYJzrg5bYj_j%5mUnn1Xtp@np_sAg==oCkfl zYYNr;_t!@sO{bsfd**u}#V7$4(GS_Z%b`>_hyZQ~pjTBz8JqgcTOG7+E}YC@iOFaJZRg%_A8rx;et5o`bQ zGJf~o{zon**#HO`>-3Lb(gq1Lm@h$ z5?@#AOkG2l^PXur&|sjW5631-dOfyumt_e{4=s+yljdVvsRX(J>kiK;I+cSr+;9gf8lXWYsa2)Q8$Ia z07`dciILB8p|M{=rt7d|m&!(b)gcWV5NDM`^!!1%ty!{G)YI@Gg5-N8lw>dZOv_BB zBAjg!CbBYmjnl9uzdPBAj>_8&GX43WwtM&@BZ8~ch{v$$hI z6W)nB#w~|ucDMFa#l?Y6U!3b8e0c=fz%Yb{=5H%W$l_iS0w#R_0iduG z*po*0NEX}B4*BeYIbpqN!iXi3FZTe4R+9JnaO1V&-*Sck!y%Hyw^yh!jx1%UF;GpH zuVny>!*EL={aQ`)IfxcQlUBs5F1D$gp+nk2A6HZwS4s1W)-9I`)jBJQLoX%ak52Blc&-H)njtcS^r81_XQ#g!61&>A)GxFtpuz(Ix zq3SZ1zq>g%#L`u_xT}I9^cEm+#$-@i0pVWPt)C@!k#!ck(`P|{&ZFa4SnJ!v@5HSH zebHd3F3BO9Y_)Iu8TJq@yNXc}x9msg5jy@v_6RN3eOM~#D!5qGmf2=nW(;Bc4SaCh z`>Ni-_Q+gM79t9G5C3BMHZ|I_N96=h@h*7Qs7Fk7V)k=Sk&wZ-{iA0Kf3>cYu2B(U z$MZ53CcGYKX=A>TiOXfv43-T|rpv6a9mJu@ z{%Zda5L4Mi;I32s%s>Z)XNY%F0>U~8^%cLzJ#>!`Z z^7)JU!&sE5kXiX`Tu;8mJ>`6vWh}#m=kS^*xaOqcu36u!RK4^p&8nKyoasSy_bRSa z5dE1So;mP|T$OL)Ov5LTuPH$-U9QYQMYF!%#2-X@T1a2{kb|r~cH5IT*UtsJlxO*{ zoRAnKDN15n6erC4#&6See@#&r)U!fe{n+ioxljBougaWc?Bc-0A0{bPA*bw97(z+> zqevRu`}%g-6XZ|t1ujRBSiO18^X(|g7?HjdUdjz$iDoqm72PdY*n9)t*1v`Xlk zOBj6wQ}38HyzUhcoH8MNkevR94qC03w%eCB?@MdGRaP`R+YDMvbe$}5|A0X+9!*`X zJ)G0IFS$&g*+`I_5Iw3?M6P35ekxIq5?0MHUo&ZRNxVq=of{rwt-L~DYI^^z$rN6_ zaGT83+}=_tW+_OKQdXRgoq{4HC2yD ziGpm(VGVJnUN>i2ibCjuHGUH2*HJs~4iqN*T4P8kvC6WM1oCF-hE}|-ZhRgLaX3X= zpW&1IR%LNy9QikT63IOKAa_cz>X#G`>ek?4e?;1*euJluVXl=kg9R|CEa%;M8XJ5t z3(Ry^5MtT$`Fyt#<7487tuTzrGQF}JG}vo|V1I!xV~5#e!nyFNJ|MN63q&vv6pDo0 zh5;|bAUcPjfzR=9niVLBnZC3$+`x9PhI+O;P`(}{Pa*gC_Xk~pof>4Y6m2n78=g)@ zjh!+DPJ--mFHbRV&oJ`U(X@Jw~H+oAC+0pKi2Mr#7_ z%yn_@WATnY0ak{}k8}sQm&j#|Av}uxtaKg98#o$k*$OHtbr?7ha6ArA=&LLE=OVn# z*Rk`3+~k|swRi4fE{}JdlaNoYy(j{XhFqNuJ$ab@@KWF2Q&CceV&pFsW4En~j~nf; zt+&xf+Rc|m3*+xf8vzhz03I|D(3Kew5W;_B#lg|d@;{+`tfk?yQjG4+-`O*Iinn3Z zQudQ}utcR!hGMv6?FMU@b3p{18b?PHiCAF?DKba1UO59jrB)^I>F?&$IHj* z6JDVswV{@S=oiczk13|<3HuX<2c5=44)@o|+taLd_nqZn?dV45wx+v0AHqH_9!%Fg zV0^z@2>V=NBagsn-znk_f}H1d<}EblXA;LD-4D3t09Y%5gT%IgUH50xdWzKPEX4H| z%+#q%%KCh~C>PMAS)bQz?P%gvqD$x#KO2#-!DW(f3hX=v{nY#|X#SJL?hzWKvdU}iypXfBf z)tsi8hYW*82|4hixwJq{)V1U2{px5pYrJs*@=sqQa($lloAN zr|bF@iL|i#z3pwtJ67(%<)I19z6cFwatf0L_H9dw#LCCZ_nk+#NR8DJ9Pg;Nphj5G zVL_Chk!&B5js*D{i;`j;_LFWIUVrsmkc_M2JDJ<*!1;6TAB`u%TMN%mBIaW#KlMV4me8S(o@jjt<9;g5zkmfm zdc|mcB|~uM6Y7f66`OI#>w#~V&oB6KnS`m}me$H?gVI3T1aDJEA%j9UD7b^tIQo)6 zK3?1QlG{onuM94Q&M@0%=>WvT-M7mvJ{ElVQxj*GLXS2xD>ZA@e3s?MsA_7@b|cjX z<$~&DT#`#KW|M(T(x>8#vwWSzvv@nEEiN^%9@zvaezWwm_>DkWd$`X%O*LOCCD-?E zhTSe?5{G%L(%WXUliFrs-L4f*WVF*-l^g3X(oCGeq9zi!xGmE2>o#yBRE%@h z*#Xztseb!h96p&$=Mu7*Tv>J5+!30TEcVmvBp99aB-fQ6yVWyz9P%kW{ORu@dj{`x zJBLjWXI5Zl`aDFV37j5%+cF5ZB|x#71Y+#ghOWZvzP2dt*)K>@*BH-$SdjCFuw`;e z3^8qr;+u_RB~UZ#eg^@3tx5o1<2<9iy(*6@GNE&Izmecw!_zTO^ek!Elsry+s09?+ z>{@53*k87RaZKuzr1Sjv)E-B2jgDK=sUYUOX;{2E9xh?6UB9{4Pvdksihr*4eaKVJ zgCpsh(u1pUq&1t2Pl2<7#-p(Du4UEMoNMgV<0kL0XmhRSY+R_hNI5$=dP4W`1uP{gSExi^0Aw8v(tIMe%h$ z->t!{9vNc1v|8DUh3d`C=ISvt|r8#tda#GnIY}b zX41&3>$j)9kxr)T6T5HfiL-96*69}|#49>ZSB+yJoFCumaO0gB14w zqZskjFR+})GEv4;Q&AglriQF3O@ijW9yum6wac2l?#QwC0=*si2AY}RRC48&>_ITv ziGjsY#ep^9l}4rnd4lRk_UvAWD2r47^X2F%UMfD00fwz*d;It-^}z+2J^=BsxIl8X zL@F^)!0CjZ{E;1L(8;w?DGlwn5ec{I4v-I3r@8HS4@Q^?iCPnK0- zSUePoUI}Lz&Be41_rMfGd-6#aGX*34Q8H@!PqNX)#3}OXF9(#z@`|Z(9@txT&=y{f zZlA>)j*qFiW8-vAtyS6!WOJSKisa`eRmr7othf989|=+D#?b9@4-Tb@WVz``72^pW zo&u2a$F0^Qu7aIp6prz~i=c?m# z)rLzD8p{;C)=VmR=U4woGVMgb7O|FLk9$ZAo+_q5yH_2wB{Y1MVfB zyBfks;Xfg$y#k4dv?DUbHAL%1(uI__XXNC2_)zU8Q)dA!D0Y2K%UjfFjd?Z@^cC!g zc$LfUb`wR41&ZiGWQJ?%ydk_hyYV0c8Hr_rlW-#^3OYE~?Av-Z^r9Gso?e7FS!DTx zJB%MV-ZDpLKQJ>ARbCV50hjLnoUIXGtYKz=Zr8nvwO%319nRZA(W4!PJQT=2 zMejY&(>uuC;&pn|0pbtA$ZC7)AGSLV^f@f_ZQ-&TT)edb;sINMmzlF8OJYYo$VPHgbmgK+m#V^Ng)TlQn&_d z9X1Y3C;a#;V(K}z`eAf~s}Ii(gZ`pY%@NQN0B0nwOA*Urn&pdv#7w(g7&~HwYynmzJPEtoRopQCTqz0<_xFKEX-JpBGJOW}|nAj6O zLoftpr)(!bM*p?!i2iVP_nc6G$>HEVauqqH5-g>{a$^Ifu{0;xhoIK+R7+|uj-eKn zm*!Qc0*z!(4+G({DHSL-oj<)uU=2W7eyhXn-lWF=Noxi+R(~mLf0N71>F%&zLPYRQ z6{vN^S>6QEv>SyRs73_B0al@bSS*y=^Tip}gIuy5Qszq^&h7Ij2m>|^{VyG8k6I+u zhj7jl8|rq?&n2Dx_l{B{gud6YWvi!Zh03r@;ACX!-Tr%c;D-(sVjy^ziPp?SPJUCW zdt=>{ooRc7f;(PjKex>a1b%W7kH|i5tp_hW2Kw>?r z?!y&-sl_E@@{t&0OzNn&QC_+&Obpa!Q4`~CSF^D-4v0~-%iDLQvSifme7T?ulgN4j z{?iXz-RBc{Z_y<7k00LT2|0TkTL%UM8+)U_C~RTe_+KJd2Ia9%aZh`Rf91{lr#VvE;swW5YAP5MRihRzNTS4i8 z%EMag1r382fA98=w%h!#BcMAUfl6qoFsUtbD-YbTRC)knhhL6MLYi6vgiYFERoe3LQ$lMG7IHY<#t*zMQx7xJ^YvRE$w~S45cT}Sc1NTYr^v4{? zw(xhS1%7V3#g*&Bqfg6$a!kjLT+8X1Cy+X(JeG*moy30;8_Ic2B%A7S2(Oe|2r`$+ zR;)zeE#-wW7eXn86)lZUD~^C~d%&&0rm71>&M>Hp^kz`eAvI}USP1=I(@vBo-rgt> z*S-l%@nN$$(sx-niG3nqS^uY2bS>vz2?jprp+FC2ca|iazfG5huHd0H8+fs#eaG|} zt;CXZg=ON6z*r?HP6p|){ie|!-g8u-r&kX&ct5PY-JCbYO^FyzZO2$IFAFjojW&TWdi1c)eY~j*-BT(e^lLahOm}-NS})v* zYM>m1`u+eOw&aK7&4S|1)Eoslewv@rpu7{=CVSR=(&USFoQ%KR06BQ;`IwIZ_78Ev zIHnRSYj48tVz(y3 zay0vj)gP#$i!z^A!M4wPY5-^(+-D9c>A2oXWq@#F@ww9cAIO*%HuZ;TCFP7Q23uPiKy7V<3 ziPc<`#0q4h132$g4zI$l>gTukN-!_XN9n1%cDtCqbN{+r;v?$7eIiNHQZBv>>FV|) zHY?#{8^hq<;Ox!yVQey%Vf=_NzI$?vD|#?4m4I_jA>pMq2}j(#Z5^ssrSFr6UX zxm9wubEf0BTgJ0vcHF8UerruKlxz(D7Q~<4D2~Hw{RQ7FwGmS>Phdto zGOueY%NdI?2gg~}Z_ODq7){j=t@UUS)U?KlUq1ntCy#d4n!A1=4HS|%ZnF2*>)tzc zVg4{01SAS^J0;ku_nRd)P!K-ermYgk$?dnGQs8FaEg)p4wI`zZ6`cBn*$*ZrPZWzPQl2k0?G#UtNG}@m$+XPTa!%(g6|Ysjv36!Dj1}9p z2>aF8MYcD0Up!pBjEZ)Ut*lWZ7MRY|F>a1pnJU>>EiLY2TG~qes2gh`Sz7Ht`I>ss z@y>KYvT4RVE8ML0f`l9E1af3++SgSxr6U>q$n?B(DEL8arG&5=BLOJ_57GV9 zg3E3-8mFhme%x;}0)tErs>qtk6;d*#hg+kVc-|gmT z=Vz5JMzm)x$&%j|rS#cJkHZrPF3zXcu>%6~7#!M?xIAkvm{Ds{hl9HFDI#P(Qg@7cXRV zy?z68)Lg0vGkuM7d{Q*(=bO+gYEB^{WWua`gFwf-FPJp@bzgWta^XRrAqJ55xeMRe z>g?pZ*b*YRmnLd`OVFQghsY5Rb?#;7Rv=F0t5Bf-1dv@N*oKj!XyJ<7SIc&j<5;sCp_DEIIztSFwBY)m&AYvXGlxOK z{BYlf2bmQJmx_cK2tFHddM81^^j_m%1CiKGz&h!WuX>t9n2<~1l!cUX8A=2rnl)Ke z3s7;w1z%?^Wke*%t%!zn8E^_fu6CAy&4ldHIFBYT+e4Vqxx?A9 zpVt!`I0VHa!r6CTU^B``e7C;}5rRX)8Q%DMtvmZYx-AOSm$925Y zlk=AQmhp#4QW!Nk@Y7}TS8FqsyUV#Vg<6!@VLlhhQ?ue089L#PT5=pym^3l=AkdnS z%VB`0NXc##Ne?8im%!~UiYN18qng?0D@t_@^E4#2KV+)=7Ge>s;FVc&94Y6{H?9gT zTek+87tT@ETQ*s77?wD+MpA%tN}y|uJGZVl9r>@s9gZ0HQn98LDtA{nr#UbFN?7@- zsriAWzP@L*&*71F%RB$8bWnc&LVjaRze4IFA)2I3UVvvEHr=6UZX(zYpGq|&6n#Ynx*(9+ zFli8t7SnE<5RQ0-2V1%T2H50Mz(ivl6^=-tCPXllxt1BF8(6be#mks^1G}4%;{cFm zl>+WUd;(1yHqjh=$giNhnQJc%HMsS2ZP7$B3;ZC0biQtCDgLjWMXiILZC80cA_?V*C~vsV|fz4UJd_tPeO#}F;A0UQ6#YEJtoa4VD zF+oKaq`*413AfuUp!?8wApHZAqCpQgIRt=MbCk(#>ZP>4M0h9_a@uSM-(=x}^h+gj zN$=Mi1>m>x{Qo~Z_x6|m_44ceK>lOukEwz8`1ifz-}vaaj`H72`Q1r=4}LEf{su3%v-j}#`qFPW1;)P>l7G~u_we_U%5S&>=D*?p6Ik9GcrR`IHb9H@U*G*J zN#DcYv)AA75S)MA{~s><9{rwa{YGQI&9(gP`}e%-y@~ganBOK0N&mNf|9arO$G<JCL-E(I{{z!fiJ|}i literal 0 HcmV?d00001 diff --git a/ci-corpus/pdf/.gitkeep b/ci-corpus/pdf/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ci-corpus/tex/figures/ball.png b/ci-corpus/tex/figures/ball.png new file mode 100644 index 0000000000000000000000000000000000000000..f37764b1f7606623616dcdc169cc858273ea2d94 GIT binary patch literal 70 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx1|;Q0k92}1TpU9xZYBRYe;|OLfu)tPp=D){ QB2a?C)78&qol`;+0Lr!y6951J literal 0 HcmV?d00001 diff --git a/ci-corpus/tex/sheet-1.tex b/ci-corpus/tex/sheet-1.tex new file mode 100644 index 0000000..3127901 --- /dev/null +++ b/ci-corpus/tex/sheet-1.tex @@ -0,0 +1,46 @@ +% Synthetic, in the shape of the corpus's problem sheets: a starred section +% heading, an enumerate of questions with lettered parts, and a solutions +% section at the end. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 1: Kinematics} + +\begin{enumerate} + +\item A ball is thrown straight up at $20\,\mathrm{m/s}$. + \begin{enumerate} + \item Find the greatest height it reaches. + \item Find its time of flight. + \end{enumerate} + +\item A block of mass $m$ rests on a slope of angle $\theta$. + \begin{enumerate} + \item Name the three forces acting on the block. + \item Find the least coefficient of friction that holds it still. + \end{enumerate} + +\end{enumerate} + +\section*{Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item $h = v^2 / 2g = 20.4\,\mathrm{m}$ + \item $t = 2v/g = 4.08\,\mathrm{s}$ + \end{enumerate} + +\item + \begin{enumerate} + \item Weight, the normal reaction, and friction along the slope. + \item $\mu = \tan\theta$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-2.tex b/ci-corpus/tex/sheet-2.tex new file mode 100644 index 0000000..f18652d --- /dev/null +++ b/ci-corpus/tex/sheet-2.tex @@ -0,0 +1,28 @@ +% Synthetic: questions with lettered parts and no solutions on the sheet. The +% solutions are in solutions-2.tex, which is a document of its own, as the +% corpus's problem sheets are. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Fields} + +\begin{enumerate} + +\item A dipole sits at the origin. + \begin{enumerate} + \item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. + \item Find the field on the axis at a distance $z$. + \end{enumerate} + +\item A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. + \begin{enumerate} + \item Find the radius of its circular path. + \item Find the period of the motion. + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-3.tex b/ci-corpus/tex/sheet-3.tex new file mode 100644 index 0000000..39d12d8 --- /dev/null +++ b/ci-corpus/tex/sheet-3.tex @@ -0,0 +1,50 @@ +% Synthetic: a sheet whose first question includes a figure, which is the shape +% that first failed a sweep on the path of an image. Nothing here is copied +% from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 3: Statics} + +\begin{enumerate} + +\item The beam below carries a load $W$ at its midpoint. + + \includegraphics[width=0.2\textwidth]{figures/ball.png} + + \begin{enumerate} + \item Find the reaction at each support. + \item Find the bending moment at the midpoint. + \end{enumerate} + +\item A ladder of mass $m$ leans against a smooth wall. + \begin{enumerate} + \item Draw the forces acting on the ladder. + \item Find the least angle at which it does not slip. + \end{enumerate} + +\end{enumerate} + +\section*{Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item $W/2$ at each support. + \item $M = WL/4$ + \end{enumerate} + +\item + \begin{enumerate} + \item The weight at the centre, the normal reaction at the wall, and the + normal reaction and friction at the floor. + \item $\tan\alpha = 1/2\mu$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/solutions-2.tex b/ci-corpus/tex/solutions-2.tex new file mode 100644 index 0000000..0cd053c --- /dev/null +++ b/ci-corpus/tex/solutions-2.tex @@ -0,0 +1,27 @@ +% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the +% shape that first failed a sweep — a document with no questions in it. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item Take the divergence term by term; each pair cancels. + \item $B = \mu_0 m / 2\pi z^3$ + \end{enumerate} + +\item + \begin{enumerate} + \item $r = mv / qB$ + \item $T = 2\pi m / qB$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml new file mode 100644 index 0000000..54afdc5 --- /dev/null +++ b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Para text~'^(STRESS ANALYSIS|Sheet |Note:)|Description automatically generated' +question: ListItem +part: Para text~'^[A-Z]' +solution: Para text~'^\[' +strip: ['^\d+\.\s+', '^\['] +layout: PartsOneSol diff --git a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml new file mode 100644 index 0000000..c4bf288 --- /dev/null +++ b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: Para +solution: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml new file mode 100644 index 0000000..a3c7db3 --- /dev/null +++ b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Para text~'Figure [0-9]:' +question: Para label~'^(1a|Q[0-9])' +part: text~'.' +strip: ['^1a\) ', '^Q[0-9]\s*', '^- ', '^[0-9]+\. ', '^i+\)\s*'] +layout: PartsOneSol diff --git a/corpus-specs/baseline.json b/corpus-specs/baseline.json new file mode 100644 index 0000000..1362a08 --- /dev/null +++ b/corpus-specs/baseline.json @@ -0,0 +1,26 @@ +{ + "specs": "corpus-specs", + "folders": { + "ci-corpus/tex": { + "root": ".", + "suffixes": [ + "tex" + ], + "built": 4 + }, + "ci-corpus/docx": { + "root": ".", + "suffixes": [ + "docx" + ], + "built": 1 + }, + "ci-corpus/pdf": { + "root": ".", + "suffixes": [ + "pdf" + ], + "built": 1 + } + } +} diff --git a/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml b/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml new file mode 100644 index 0000000..97c98ce --- /dev/null +++ b/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Header +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml b/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml new file mode 100644 index 0000000..2a668e6 --- /dev/null +++ b/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: ListItem text~'^[A-Z]' +solution: after Header text=Solutions, ListItem +strip: ['^\d+\. '] +layout: PartsOneSol diff --git a/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml b/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml new file mode 100644 index 0000000..1754146 --- /dev/null +++ b/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml @@ -0,0 +1,4 @@ +ignore: Header +question: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index fcde4e6..4f3f9c9 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -3,10 +3,11 @@ import argparse import getpass import sys +import tempfile from pathlib import Path from typing import Optional, Sequence -from in2lambda_agent import compare, corpus, pipeline +from in2lambda_agent import compare, corpus, gate, pipeline from in2lambda_agent.mathpix import MathpixClient, MathpixError from in2lambda_agent.model import ModelUnavailable, choose_backend from in2lambda_agent.ocr import ocr_pdf @@ -67,7 +68,8 @@ def build_parser() -> argparse.ArgumentParser: """The command line as the design spec describes it. Returns: - A parser with the `run`, `review`, `corpus` and `compare` subcommands. + A parser with the `run`, `review`, `corpus`, `gate` and `compare` + subcommands. """ parser = argparse.ArgumentParser( prog="in2lambda-agent", @@ -199,6 +201,37 @@ def build_parser() -> argparse.ArgumentParser: default=corpus.DEFAULT_SPEC_DIR, help="The tree the sets' specs are kept in, mirroring the corpus.", ) + sweep.add_argument( + "--cache", + type=Path, + default=pipeline.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, so a sweep pointed at a cache " + "another run filled converts nothing.", + ) + + check = subcommands.add_parser( + "gate", help="Replay the corpus the baseline names and check it against it." + ) + check.add_argument("baseline", type=Path, help="The committed baseline file.") + check.add_argument( + "--record", + action="store_true", + help="Write this run's counts to the baseline instead of checking them.", + ) + check.add_argument( + "--cache", + type=Path, + default=gate.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, shared between worktrees so " + "that a conversion is paid for once.", + ) + check.add_argument( + "--work", + type=Path, + default=None, + help="Where the folders are copied to be run, under the system temp " + "directory by default so the check writes nothing where it was run.", + ) against = subcommands.add_parser( "compare", help="Check a PDF's OCR against the pages it came from." @@ -241,6 +274,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: specs=args.specs, replay=args.replay, rounds=args.rounds, + cache=args.cache, settings=load_settings(), ) print(f"{len(rows)} documents, written to {args.results}") @@ -249,6 +283,27 @@ def main(argv: Optional[Sequence[str]] = None) -> int: succeeded = {"built", "skipped"} return 0 if rows and all(row.outcome in succeeded for row in rows) else 1 + if args.command == "gate": + baseline = gate.read_baseline(args.baseline) + # The directory is printed and is not deleted, so that the drafts of a + # folder that failed can be read after the run. + work = args.work or Path(tempfile.mkdtemp(prefix="in2lambda-agent-gate-")) + print(f"work {work}") + report = gate.run( + baseline, + record=args.record, + cache=args.cache, + work=work, + settings=load_settings(), + ) + for name, summary in report.folders.items(): + print(gate.folder_line(name, summary)) + if args.record: + gate.write_baseline(baseline, args.baseline) + print(f"recorded {args.baseline}") + return 0 + return 1 if report.failed else 0 + if args.command == "compare": settings = load_settings() try: diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 95fb16e..7ff1f46 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -249,6 +249,7 @@ def run_one( settings: Settings, rounds: int = 3, replay: bool = False, + cache: Path = pipeline.DEFAULT_CACHE_DIR, backend: Optional[Backend] = None, ) -> Row: """Runs the pipeline over one document and reads the row off what it did. @@ -265,6 +266,7 @@ def run_one( settings: The environment the run has available. rounds: The round limit, ignored in a replay, which can run none. replay: Run the saved spec and nothing else, making no model call. + cache: Where the OCR of each PDF is kept. backend: The backend to write a spec with, chosen from the settings if absent. @@ -285,6 +287,7 @@ def run_one( # report: the row then says what the saved spec left rather than # that a call could not be made. rounds=0 if replay else rounds, + cache_dir=cache, backend=NoModel() if replay else backend, ) except ModelUnavailable as error: @@ -362,6 +365,7 @@ def sweep( specs: Path = DEFAULT_SPEC_DIR, replay: bool = False, rounds: int = 3, + cache: Path = pipeline.DEFAULT_CACHE_DIR, settings: Optional[Settings] = None, backend: Optional[Backend] = None, ) -> list[Row]: @@ -376,6 +380,8 @@ def sweep( specs: The tree the sets' specs are kept in, mirroring the corpus. replay: Run the saved specs and nothing else, making no model call. rounds: The round limit each run is given. + cache: Where the OCR of each PDF is kept, so that a sweep pointed at a + cache another run filled converts nothing. settings: The environment the runs have available. backend: The backend to write the specs with, chosen from the settings if absent. @@ -455,6 +461,7 @@ def sweep( settings=settings, rounds=rounds, replay=replay, + cache=cache, backend=backend, ) print(f"{row.outcome:<20} {row.source}") diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py new file mode 100644 index 0000000..97c0a0e --- /dev/null +++ b/in2lambda_agent/gate.py @@ -0,0 +1,219 @@ +"""The merge gate: a replay over real documents, compared with a baseline. + +Every ticket before this one was tested on synthetic fixtures. The first sweep +over a folder of real PDFs failed on four faults no fixture had: a list-valued +selector, a file of solutions with no questions, an image path, and a wrapped +line. A merge now requires an end-to-end run over real documents. + +The run is `corpus.sweep(replay=True)` over the specs the repository keeps, so +it makes no model call. The baseline file names the folders to run and records, +per folder, how many documents built. The gate fails when a folder builds fewer +documents than the baseline records. A folder the baseline records no count for +passes on any count, which is how a folder is added to the gate before it +replays to a build worth defending. The baseline is committed, and a change to +it belongs in a pull request that states why the counts changed. +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from in2lambda_agent import corpus +from in2lambda_agent.settings import Settings + +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "in2lambda-agent" +"""Where the gate reads the OCR of each PDF. The directory is outside every +worktree, so a PDF converted on one branch is not converted again on the next. +`corpus` on its own keeps its cache under the directory the user ran from.""" + + +@dataclass +class Folder: + """One folder of a corpus, as the baseline holds it. + + Attributes: + root: The corpus directory the folder is under. The repository itself + for `ci-corpus`, which is committed; an absolute path for a corpus + outside the repository. + suffixes: The file suffixes that are documents in the folder: `pdf` for + a folder of scans, `tex` or `docx` for sources. + built: How many documents built when the baseline was recorded, or None + where no count is recorded. `root` and `suffixes` are written by + hand; `built` is what `--record` writes. + """ + + root: Path + suffixes: list[str] + built: Optional[int] = None + + +@dataclass +class Baseline: + """The committed file the gate compares a sweep against. + + Attributes: + specs: The tree the folders' specs are kept in, relative to the + repository. Folder `A/B` reads `/A/B/in2lambda-spec.yaml`. + folders: The folders to run, by their path under their own root. + """ + + specs: Path + folders: dict[str, Folder] + + +@dataclass +class Summary: + """What one folder did this run, beside what the baseline records. + + Attributes: + built: How many documents built. + counts: How many did each other thing, by outcome. + recorded: How many built when the baseline was recorded, or None where + no count is recorded. + """ + + built: int = 0 + counts: dict[str, int] = field(default_factory=dict) + recorded: Optional[int] = None + + @property + def failed(self) -> bool: + """Whether this folder fails the gate.""" + return self.recorded is not None and self.built < self.recorded + + +@dataclass +class Report: + """The whole run: one summary per folder, in the baseline's order.""" + + folders: dict[str, Summary] = field(default_factory=dict) + + @property + def failed(self) -> bool: + """Whether any folder fails the gate.""" + return any(one.failed for one in self.folders.values()) + + +def read_baseline(path: Path) -> Baseline: + """Reads the committed baseline. + + Args: + path: The JSON file. + + Returns: + The baseline. + """ + written = json.loads(Path(path).read_text(encoding="utf-8")) + return Baseline( + specs=Path(written["specs"]), + folders={ + name: Folder( + root=Path(one["root"]), + suffixes=list(one["suffixes"]), + built=one.get("built"), + ) + for name, one in written["folders"].items() + }, + ) + + +def write_baseline(baseline: Baseline, path: Path) -> None: + """Writes the baseline back, counts and all. + + Args: + baseline: What to write. + path: The JSON file, which is overwritten. + """ + written = { + "specs": baseline.specs.as_posix(), + "folders": { + name: { + "root": one.root.as_posix(), + "suffixes": one.suffixes, + "built": one.built, + } + for name, one in baseline.folders.items() + }, + } + Path(path).write_text(json.dumps(written, indent=2) + "\n", encoding="utf-8") + + +def run( + baseline: Baseline, + *, + record: bool = False, + cache: Path, + work: Path, + settings: Optional[Settings] = None, +) -> Report: + """Replays every folder the baseline names and counts what each one built. + + Nothing is written into the repository or into a corpus: each folder is + swept into its own directory under `work`, and the table is written there. + + Args: + baseline: The folders to run and the counts to compare against. In + record mode this run's counts replace them. + record: Take this run as the new baseline rather than checking it. + cache: Where the OCR of each PDF is kept. The directory is shared + between worktrees, so Mathpix converts each PDF once. + work: Where the folders are copied to be run. + settings: The environment the runs have available. + + Returns: + One summary per folder. + """ + work = Path(work) + report = Report() + for name, folder in baseline.folders.items(): + rows = corpus.sweep( + folder.root, + paths=[Path(name)], + suffixes=folder.suffixes, + results=work / name / "results.csv", + work=work / "work", + specs=baseline.specs, + replay=True, + cache=cache, + settings=settings, + ) + summary = Summary(recorded=folder.built) + for row in rows: + if row.outcome == "built": + summary.built += 1 + else: + summary.counts[row.outcome] = summary.counts.get(row.outcome, 0) + 1 + if record: + folder.built = summary.built + # A recording run reports what this sweep did, so it fails nothing. + summary.recorded = summary.built + report.folders[name] = summary + return report + + +def folder_line(name: str, summary: Summary) -> str: + """The gate's one line for a folder: what it did, beside what is recorded. + + Args: + name: The folder, as the baseline names it. + summary: What it did. + + Returns: + The line. + """ + counts = " ".join( + f"{outcome} {summary.counts.get(outcome, 0)}" + for outcome in ("faulted", "build refused", "skipped") + ) + other = sorted(set(summary.counts) - {"faulted", "build refused", "skipped"}) + recorded = ( + "not recorded" + if summary.recorded is None + else f"baseline built {summary.recorded}" + ) + return ( + f"{name:<40} built {summary.built} {counts}" + + "".join(f" {outcome} {summary.counts[outcome]}" for outcome in other) + + f" ({recorded})" + ) diff --git a/in2lambda_agent/ocr.py b/in2lambda_agent/ocr.py index b34f088..707ad2c 100644 --- a/in2lambda_agent/ocr.py +++ b/in2lambda_agent/ocr.py @@ -9,6 +9,7 @@ import shutil from dataclasses import dataclass from pathlib import Path +from typing import Optional from in2lambda_agent.mathpix import MathpixClient @@ -25,6 +26,27 @@ class OcrResult: fresh: bool +def cached(pdf: Path, cache_dir: Path) -> Optional[OcrResult]: + """The conversion already in the cache, or None where there is none. + + Asked before a client is built, so that a document whose OCR was fetched + once runs again with no Mathpix credentials at all: a worktree, or a CI job + on a fork, has the cache and not the account. + + Args: + pdf: The PDF whose conversion is wanted. + cache_dir: Holds one entry per document, named by the PDF's hash. + + Returns: + Where the markdown and its media folder are, or None. + """ + entry = Path(cache_dir) / _hash(pdf) + markdown = entry / SOURCE_NAME + if not markdown.exists(): + return None + return OcrResult(markdown, entry / MEDIA_NAME, fresh=False) + + def ocr_pdf( pdf: Path, *, cache_dir: Path, client: MathpixClient, fresh: bool = False ) -> OcrResult: @@ -42,11 +64,12 @@ def ocr_pdf( Raises: MathpixError: If the conversion fails; the entry is left absent. """ + if not fresh and (hit := cached(pdf, cache_dir)) is not None: + return hit + entry = Path(cache_dir) / _hash(pdf) markdown = entry / SOURCE_NAME media = entry / MEDIA_NAME - if markdown.exists() and not fresh: - return OcrResult(markdown, media, fresh=False) # A fresh pass restarts the pipeline for this document, so the whole entry # goes: anything a later stage comes to keep beside source.md — a draft, a diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index 7045a03..55dbcfc 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -34,7 +34,7 @@ from in2lambda_agent.fix import RoundResult, fix_round, summary from in2lambda_agent.mathpix import MathpixClient from in2lambda_agent.model import Backend, ModelUnavailable, Usage, choose_backend -from in2lambda_agent.ocr import ocr_pdf +from in2lambda_agent.ocr import cached, ocr_pdf from in2lambda_agent.review import RECORD, Question, Review, choose from in2lambda_agent.settings import Settings from in2lambda_agent.spec import RECORD_NAME, record_run, spec_path, write_spec @@ -125,7 +125,8 @@ def run( Raises: MathpixError: If a PDF cannot be converted, MissingCredentials among - them when the run has no Mathpix credentials. + them when a conversion is needed and the run has no Mathpix + credentials. A PDF already in the cache needs none. ModelUnavailable: If a spec must be written and no backend can run. BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. @@ -145,8 +146,13 @@ def run( # The rest of the pipeline reads markdown, so a PDF becomes markdown first. if source.suffix.lower() == ".pdf": - client = mathpix or MathpixClient.from_settings(settings) - ocr = ocr_pdf(source, cache_dir=cache_dir, client=client, fresh=fresh_ocr) + # The cache is asked before the client is built: a document converted + # once runs again with no credentials, which is what lets a worktree or + # a fork's CI job replay a corpus of PDFs it cannot pay for. + ocr = None if fresh_ocr else cached(source, cache_dir) + if ocr is None: + client = mathpix or MathpixClient.from_settings(settings) + ocr = ocr_pdf(source, cache_dir=cache_dir, client=client, fresh=fresh_ocr) frozen = ocr.markdown # A fresh pass is a restart: every stage below reads the new markdown. message = ( diff --git a/tests/test_cli.py b/tests/test_cli.py index 740d234..5020c2d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,7 +7,7 @@ import pytest from conftest import FakeMathpix -from in2lambda_agent import cli, compare, corpus, pipeline +from in2lambda_agent import cli, compare, corpus, gate, pipeline from in2lambda_agent.cli import build_parser, main, reviewer_name from in2lambda_agent.model import Usage from in2lambda_agent.settings import Settings @@ -90,6 +90,7 @@ def test_corpus_defaults(): assert args.results == Path("results.csv") assert args.work == Path(".in2lambda-agent/corpus") assert args.specs == Path("corpus-specs") + assert args.cache == Path(".in2lambda-agent") def test_corpus_every_option(): @@ -112,6 +113,8 @@ def test_corpus_every_option(): "working", "--specs", "saved", + "--cache", + "cached", ] ) @@ -122,6 +125,104 @@ def test_corpus_every_option(): assert args.results == Path("sweep.csv") assert args.work == Path("working") assert args.specs == Path("saved") + assert args.cache == Path("cached") + + +def test_the_corpus_cache_is_handed_to_the_sweep(monkeypatch): + given = {} + + def record(*args, **kwargs): + given.update(kwargs) + return [] + + monkeypatch.setattr(corpus, "sweep", record) + + main(["corpus", "ExampleContents", "--cache", "cached"]) + + assert given["cache"] == Path("cached") + + +def test_gate_defaults(): + args = build_parser().parse_args(["gate", "corpus-specs/baseline.json"]) + + assert args.command == "gate" + assert args.baseline == Path("corpus-specs/baseline.json") + assert args.record is False + assert args.cache == Path.home() / ".cache" / "in2lambda-agent" + # Chosen when the command runs, so that two runs do not share a directory. + assert args.work is None + + +def test_gate_every_option(): + args = build_parser().parse_args( + [ + "gate", + "saved.json", + "--record", + "--cache", + "cached", + "--work", + "working", + ] + ) + + assert args.record is True + assert args.cache == Path("cached") + assert args.work == Path("working") + + +def test_a_gate_that_passes_exits_zero(tmp_path, monkeypatch, capsys): + path = written_baseline(tmp_path) + report = gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + monkeypatch.setattr(gate, "run", lambda *args, **kwargs: report) + + code = main(["gate", str(path)]) + + assert code == 0 + assert "tex" in capsys.readouterr().out + + +def test_a_gate_that_fails_exits_one_and_says_what_the_folder_built( + tmp_path, monkeypatch, capsys +): + path = written_baseline(tmp_path) + summary = gate.Summary(built=1, counts={"faulted": 1}, recorded=2) + monkeypatch.setattr( + gate, "run", lambda *args, **kwargs: gate.Report(folders={"tex": summary}) + ) + + code = main(["gate", str(path)]) + + assert code == 1 + assert "tex" in capsys.readouterr().out + + +def test_recording_writes_the_baseline_and_exits_zero(tmp_path, monkeypatch): + path = written_baseline(tmp_path) + + def record(baseline, **kwargs): + baseline.folders["tex"].built = 2 + return gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + + monkeypatch.setattr(gate, "run", record) + + code = main(["gate", str(path), "--record"]) + + assert code == 0 + assert gate.read_baseline(path).folders["tex"].built == 2 + + +def written_baseline(tmp_path): + """A baseline file on disk, for the gate command to read.""" + path = tmp_path / "baseline.json" + gate.write_baseline( + gate.Baseline( + specs=Path("corpus-specs"), + folders={"tex": gate.Folder(root=tmp_path / "corpus", suffixes=["tex"])}, + ), + path, + ) + return path def test_compare_defaults(): diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 70a1376..99bf18f 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -98,6 +98,22 @@ def test_the_row_says_what_the_spec_made_of_the_document(root, tmp_path): assert sheet.model_seconds == 0.0 and sheet.wall_seconds > 0 +def test_the_sweeps_cache_is_where_each_run_looks_for_its_ocr( + root, tmp_path, monkeypatch +): + given = [] + + def record(source, **kwargs): + given.append(kwargs["cache_dir"]) + raise RuntimeError("as far as this goes") + + monkeypatch.setattr(pipeline, "run", record) + + rows = sweep(root, tmp_path, cache=tmp_path / "shared") + + assert given == [tmp_path / "shared"] * len(rows) + + def test_a_saved_spec_and_its_source_replay_with_no_model_call(root, tmp_path): for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): saved = tmp_path / "specs" / folder / SPEC_NAME diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..d27aad2 --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,198 @@ +"""The merge gate: a replay over a corpus, checked against a recorded baseline.""" + +from pathlib import Path + +import pytest +from test_corpus import make_set +from test_pipeline import SPEC, TEX_SPEC + +from in2lambda_agent import gate +from in2lambda_agent.gate import Baseline, Folder +from in2lambda_agent.settings import Settings +from in2lambda_agent.spec import SPEC_NAME + + +@pytest.fixture +def corpus_root(tmp_path): + """A corpus of two folders, with a saved spec each so a replay builds.""" + made = tmp_path / "corpus" + make_set(made, "sheets", ["sheet.md", "sheet-2.md"]) + make_set(made, "tex", ["tex-sheet.tex", "tex-sheet-2.tex"]) + for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): + saved = tmp_path / "specs" / folder / SPEC_NAME + saved.parent.mkdir(parents=True) + saved.write_text(text) + return made + + +@pytest.fixture +def baseline(corpus_root, tmp_path): + """The baseline as it is written by hand, before anything is recorded.""" + return Baseline( + specs=tmp_path / "specs", + folders={ + "sheets": Folder(root=corpus_root, suffixes=["md"]), + "tex": Folder(root=corpus_root, suffixes=["tex"]), + }, + ) + + +def run(baseline, tmp_path, **kwargs): + """A gate run with its cache and work directory under tmp_path.""" + return gate.run( + baseline, + cache=tmp_path / "cache", + work=tmp_path / "gate", + settings=Settings(), + **kwargs, + ) + + +def test_recording_fills_the_counts_and_leaves_what_was_written_by_hand( + baseline, corpus_root, tmp_path +): + report = run(baseline, tmp_path, record=True) + + assert not report.failed + assert baseline.folders["sheets"].built == 2 + assert baseline.folders["tex"].built == 2 + # The hand-written half is the run's to read, not to write over. + assert baseline.folders["tex"].root == corpus_root + assert baseline.folders["tex"].suffixes == ["tex"] + + +def test_a_recorded_baseline_passes_the_run_that_recorded_it(baseline, tmp_path): + run(baseline, tmp_path, record=True) + + report = run(baseline, tmp_path) + + assert not report.failed + assert [one.built for one in report.folders.values()] == [2, 2] + + +def test_a_folder_that_builds_fewer_than_recorded_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built += 1 + + report = run(baseline, tmp_path) + + assert report.failed + assert report.folders["tex"].failed and not report.folders["sheets"].failed + + +def test_a_folder_whose_documents_no_longer_build_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # The spec a replay has nothing to replay without. + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert report.failed + summary = report.folders["tex"] + assert (summary.built, summary.recorded) == (0, 2) + assert summary.counts == {"no spec": 2} + + +def test_a_folder_the_baseline_records_no_count_for_passes(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # A folder added to the gate before it replays to a build worth defending. + baseline.folders["tex"].built = None + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["tex"].built == 0 + + +def test_a_folder_that_builds_more_than_recorded_passes(baseline, corpus_root, tmp_path): + run(baseline, tmp_path, record=True) + (corpus_root / "sheets" / "sheet-3.md").write_text( + (corpus_root / "sheets" / "sheet.md").read_text() + ) + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["sheets"].built == 3 + + +def test_the_folder_line_says_what_each_outcome_came_to(baseline, tmp_path): + run(baseline, tmp_path, record=True) + summary = run(baseline, tmp_path).folders["sheets"] + + line = gate.folder_line("sheets", summary) + + assert line.split() == [ + "sheets", + "built", + "2", + "faulted", + "0", + "build", + "refused", + "0", + "skipped", + "0", + "(baseline", + "built", + "2)", + ] + + +def test_the_folder_line_names_an_outcome_of_its_own(baseline, tmp_path): + summary = gate.Summary(built=1, counts={"no spec": 2}, recorded=3) + + line = gate.folder_line("tex", summary) + + assert "no spec 2" in line and "(baseline built 3)" in line + + +def test_the_folder_line_says_where_no_count_is_recorded(baseline, tmp_path): + line = gate.folder_line("tex", gate.Summary(built=1)) + + assert line.endswith("(not recorded)") + + +def test_the_baseline_survives_being_written_and_read(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built = None + path = tmp_path / "baseline.json" + + gate.write_baseline(baseline, path) + read = gate.read_baseline(path) + + assert read == baseline + + +def test_a_run_writes_nothing_under_the_directory_it_was_run_from( + baseline, tmp_path, monkeypatch +): + ran_from = tmp_path / "empty" + ran_from.mkdir() + monkeypatch.chdir(ran_from) + + run(baseline, tmp_path) + + assert list(ran_from.iterdir()) == [] + + +def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): + # Shared between worktrees on purpose: OCR already fetched for a PDF is not + # fetched again on the next branch. + assert gate.DEFAULT_CACHE_DIR == Path.home() / ".cache" / "in2lambda-agent" + + +def test_the_committed_baseline_names_the_ci_corpus_and_its_specs(): + repository = Path(__file__).resolve().parent.parent + baseline = gate.read_baseline(repository / "corpus-specs" / "baseline.json") + + assert set(baseline.folders) == { + "ci-corpus/tex", + "ci-corpus/docx", + "ci-corpus/pdf", + } + for name, folder in baseline.folders.items(): + # The spec each folder replays, at the path the sweep reads it from. + assert (repository / baseline.specs / name / SPEC_NAME).is_file() + assert folder.built is not None diff --git a/tests/test_ocr.py b/tests/test_ocr.py index cbb1355..97caafc 100644 --- a/tests/test_ocr.py +++ b/tests/test_ocr.py @@ -8,7 +8,7 @@ import pytest from in2lambda_agent.mathpix import MathpixError -from in2lambda_agent.ocr import ocr_pdf +from in2lambda_agent.ocr import cached, ocr_pdf from conftest import FakeMathpix @@ -95,6 +95,19 @@ def test_the_markdown_is_written_as_utf8_under_any_locale(pdf, tmp_path): assert written.decode("utf-8") == "# Ångström ½\n" +def test_a_document_not_in_the_cache_is_no_hit(pdf, tmp_path): + assert cached(pdf, tmp_path / "cache") is None + + +def test_a_cached_document_is_a_hit_without_a_client(pdf, tmp_path): + written = ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix()) + + hit = cached(pdf, tmp_path / "cache") + + assert hit is not None and not hit.fresh + assert (hit.markdown, hit.media) == (written.markdown, written.media) + + def test_each_document_gets_its_own_entry(pdf, tmp_path): other = tmp_path / "other.pdf" other.write_bytes(b"%PDF-1.4 a different sheet") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 1b50a15..5639b29 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -14,6 +14,7 @@ from in2lambda_agent import package, pipeline from in2lambda_agent.cli import main from in2lambda_agent.model import ModelUnavailable +from in2lambda_agent.ocr import ocr_pdf from in2lambda_agent.package import SpecRejected from in2lambda_agent.review import ReviewError from in2lambda_agent.settings import Settings @@ -1255,6 +1256,26 @@ def test_a_second_run_over_the_same_pdf_uses_the_cache(pdf, tmp_path): assert len(backend.calls) == 1 +def test_a_cached_pdf_runs_with_no_mathpix_credentials(pdf, tmp_path, monkeypatch): + monkeypatch.delenv("MATHPIX_APP_ID", raising=False) + monkeypatch.delenv("MATHPIX_API_KEY", raising=False) + ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix(SOURCE.read_text())) + + # No client at all: the cache is what the run reads, and building one from + # empty settings would raise before it got there. + result = pipeline.run( + pdf, + out_dir=tmp_path / "out", + settings=Settings(), + cache_dir=tmp_path / "cache", + backend=FakeBackend(SPEC), + ) + ocr = next(stage for stage in result.stages if stage.name == "ocr") + + assert ocr.message.startswith("cached ") + assert result.zip_path.exists() + + def test_a_pdf_without_credentials_exits_one_naming_the_variables( pdf, tmp_path, monkeypatch, capsys ): From f54e469badb2af76e0b12e80cde0b28350f356c4 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 01:34:01 +0100 Subject: [PATCH 2/6] implement: Gate every merge on a replay sweep over the corpus (t21) --- in2lambda_agent/gate.py | 15 +++++++++------ tests/test_gate.py | 4 +++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py index 8d5ebc4..35ef428 100644 --- a/in2lambda_agent/gate.py +++ b/in2lambda_agent/gate.py @@ -32,10 +32,11 @@ worktree, so a PDF converted on one branch is not converted again on the next. `corpus` on its own keeps its cache under the directory the user ran from.""" -RANK = {"built": 0, "skipped": 0, "build refused": 1, "faulted": 2} -"""How bad each outcome is. A document that did not run at all — a spec gone, a -file unreadable, a document the sweep no longer finds — is worse than any of -these, and ranks below the table.""" +RANK = {"built": 0, "build refused": 1, "faulted": 2, "skipped": 3} +"""How bad each outcome is. `skipped` is the worst of them: a document that +faults was at least read, and one that is skipped was not. Anything else is a +document that did not run either — a spec gone, a file unreadable, a document +the sweep no longer finds — and ranks with `skipped`.""" MISSING = "missing" """What a recorded document the sweep no longer finds is compared as.""" @@ -44,8 +45,10 @@ def worse(current: str, recorded: str) -> bool: """Whether a document did worse this run than the baseline records. - `built` and `skipped` are the same rank: a document in no suffix the folder - runs is not a document that stopped building. + A document that stops being read is a regression whatever it did before, + which is the case a baseline of nothing but faults rests on: widen what + counts as a solutions file and a document leaves the sweep as `skipped` + without a single count changing. Args: current: What the document did this run. diff --git a/tests/test_gate.py b/tests/test_gate.py index a3d4ab1..6dd7921 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -167,7 +167,9 @@ def test_a_baseline_of_no_builds_still_notices_a_document_that_did_worse( [ ("built", "built", False), ("built", "faulted", False), - ("skipped", "built", False), + ("skipped", "built", True), + # A document read and faulted, then not read at all. + ("skipped", "faulted", True), ("build refused", "built", True), ("faulted", "built", True), ("faulted", "build refused", True), From e8b19d5ffd02bc1159e97996fa26a35a3bd087a0 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 02:09:16 +0100 Subject: [PATCH 3/6] implement: Gate every merge on a replay sweep over the corpus (t21) --- .github/branch-protection.json | 9 --- .github/workflows/gate.yml | 8 +-- .gitignore | 4 ++ README.md | 63 ++++++++++--------- ci-baseline.json | 26 ++++---- .../specs}/docx/in2lambda-spec.yaml | 0 .../specs}/pdf/in2lambda-spec.yaml | 0 .../specs}/tex/in2lambda-spec.yaml | 0 .../in2lambda-spec.yaml | 6 -- .../in2lambda-spec.yaml | 5 -- corpus-specs/UCL_MechEng/in2lambda-spec.yaml | 5 -- gate-baseline.json | 50 --------------- tests/test_cli.py | 4 +- tests/test_gate.py | 48 ++++++-------- 14 files changed, 75 insertions(+), 153 deletions(-) delete mode 100644 .github/branch-protection.json rename {corpus-specs/ci-corpus => ci-corpus/specs}/docx/in2lambda-spec.yaml (100%) rename {corpus-specs/ci-corpus => ci-corpus/specs}/pdf/in2lambda-spec.yaml (100%) rename {corpus-specs/ci-corpus => ci-corpus/specs}/tex/in2lambda-spec.yaml (100%) delete mode 100644 corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml delete mode 100644 corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml delete mode 100644 corpus-specs/UCL_MechEng/in2lambda-spec.yaml delete mode 100644 gate-baseline.json diff --git a/.github/branch-protection.json b/.github/branch-protection.json deleted file mode 100644 index 2d1362c..0000000 --- a/.github/branch-protection.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "required_status_checks": { - "strict": false, - "contexts": ["gate"] - }, - "enforce_admins": false, - "required_pull_request_reviews": null, - "restrictions": null -} diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 5838aba..1ff0647 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -1,9 +1,9 @@ # The merge gate. Every push to main and every pull request runs the tests and # then replays the three folders of ci-corpus, the committed corpus, which -# ci-baseline.json names. gate-baseline.json names the folders of the private -# corpus, which a runner cannot read, and is replayed by the workbench check. -# The job is a required status check on main, so a branch that breaks a build -# cannot be merged. +# ci-baseline.json names. corpus-specs/gate-baseline.json names the folders of +# the private corpus, which a runner cannot read, and the workbench check +# replays those. This job reports on a branch; the workbench check is what +# holds the merge. name: gate on: diff --git a/.gitignore b/.gitignore index ea71284..fdcb1ae 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ __pycache__/ dist/ # in2lambda's KaTeX converter writes this into the working directory on import. log +# The specs for ExampleContents, and the baseline recorded over it. Both quote +# private documents: a spec quotes their headings and the baseline names their +# files. The specs for the committed corpus are in ci-corpus/specs. +corpus-specs/ # The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex # in the workflow, so the repository holds the tex source and not the output. ci-corpus/pdf/*.pdf diff --git a/README.md b/README.md index 2fee31f..efa58f8 100644 --- a/README.md +++ b/README.md @@ -285,9 +285,9 @@ review, rejections where its value comes from. One document that fails is one row and not the end of the sweep, and a set whose folder cannot be copied is a row for each of its documents. -`--cache` (default `./.in2lambda-agent`) is where the OCR of each PDF is kept. A sweep -pointed at a cache that an earlier run filled makes no Mathpix call, and needs no -Mathpix credentials. +`--cache` (default `./.in2lambda-agent`, the directory `run` caches into) is where the +OCR of each PDF is kept. A sweep pointed at a cache that an earlier run filled makes no +Mathpix call, and needs no Mathpix credentials. ## Gate @@ -307,20 +307,25 @@ still reports the document that stops being read. ``` work /tmp/in2lambda-agent-gate-3f1a -UCL_MechEng built 0 faulted 0 build refused 0 skipped 1 no spec 2 (baseline built 0) - worse UCL_MechEng/Worksheet_2.pdf faulted -> no spec: replay: no model call is allowed +tex built 2 faulted 0 build refused 0 skipped 0 no spec 2 (baseline built 4) + worse tex/sheet-3.tex built -> no spec: replay: no model call is allowed ``` Run the command from the repository root. The gate reads `specs`, and a folder's `root` where `root` is relative, from the directory the command runs in. -The repository holds two baselines, because a clone holds the second corpus and not the -first: +There are two corpora and a baseline for each: -| File | Corpus | Run by | +| Baseline | Corpus | Run by | | --- | --- | --- | -| `gate-baseline.json` | the three folders of `ExampleContents`, which is private | the workbench check | -| `ci-baseline.json` | the three folders of `ci-corpus`, which is committed | `.github/workflows/gate.yml` | +| `corpus-specs/gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | +| `ci-baseline.json` | the three folders of `ci-corpus` | `.github/workflows/gate.yml` | + +`ci-corpus` is synthetic, so the repository holds the corpus, its specs in +`ci-corpus/specs/` and `ci-baseline.json`. `ExampleContents` is a set of private +documents, and a spec quotes the headings of one and a baseline names their files, so +`corpus-specs/` is in `.gitignore` and `corpus-specs/gate-baseline.json` is written +beside the specs it names. Write both before running the local gate. Each folder's `root` and `suffixes` are written by hand. `built`, the count of documents that built, and `documents`, the outcome of each single document, are what `--record` @@ -328,14 +333,16 @@ writes. A folder the file records no `built` for passes on any count, and its li `(not recorded)`. A change to a recorded count or outcome belongs in a pull request that says why the count or the outcome changed. -`gate-baseline.json` records 0 built for all three folders. Every document of -`ExampleContents` replays to `faulted`, because pandoc's line wrapping is reported as a -math delimiter error and each document needs a fixing round that a replay does not run. -The recorded outcomes are what the gate defends until a later ticket raises the count. +Over `ExampleContents` today, every document replays to `faulted` and the baseline +records 0 built for all three folders: pandoc's line wrapping is reported as a math +delimiter error, and each document needs a fixing round that a replay does not run. The +recorded outcomes are what the gate defends until a later ticket raises the count. -`--cache` defaults to `~/.cache/in2lambda-agent`, outside any worktree, so that a PDF -converted on one branch is not converted again on the next. `--work` defaults to a new -directory under the system temp directory, which the gate does not delete: read the +`--cache` defaults to `~/.cache/in2lambda-agent`, and not to the `./.in2lambda-agent` +that `run` and `corpus` cache into, because the gate runs in a worktree of its own: a +PDF converted on one branch is converted again on the next if the cache sits in the +branch's directory. `--work` defaults to a new directory under the system temp +directory, which the gate does not delete: read the drafts of a folder that failed there. The gate also copies the spec tree into the work directory and replays the copy, because a sweep appends a record of each run beside the spec it reads. Neither directory is inside the repository, so `git status` after a gate @@ -347,19 +354,17 @@ workflow stores the markdown in the Actions cache under the PDF's hash. The job two repository secrets, `MATHPIX_APP_ID` and `MATHPIX_API_KEY`. A pull request from a fork is given neither secret. `actions/cache` restores the cache -of the base branch for a fork, and the cached markdown is what `ci-corpus/pdf` then +of the base branch for a fork, and the cached markdown is what the `pdf` folder then replays, so the job passes without the secrets. If the cache is empty — the PDF's bytes -changed, or GitHub evicted the entry — Mathpix cannot be called, `ci-corpus/pdf` builds -0 against a recorded 1, the job fails and the pull request cannot be merged. A -maintainer merges that branch by pushing it to a branch of this repository, where the -secrets are read. - -The `gate` job is a required status check on `main`: - -```sh -gh api -X PUT repos/{owner}/{repo}/branches/main/protection \ - --input .github/branch-protection.json -``` +changed, or GitHub evicted the entry — Mathpix cannot be called and the `pdf` folder +builds 0 against a recorded 1, so the job fails. Push that branch to a branch of this +repository, where the secrets are read, and the job runs Mathpix once. + +The `gate` job reports on a branch; it does not hold the merge. The workbench merges a +branch as soon as its own check passes, and its own check runs +`in2lambda-agent gate corpus-specs/gate-baseline.json` over `ExampleContents` after the +tests. `ExampleContents` is the larger corpus of the two, so the local gate is the +stricter check. ## Docker diff --git a/ci-baseline.json b/ci-baseline.json index ebd5e3b..2d0e4c8 100644 --- a/ci-baseline.json +++ b/ci-baseline.json @@ -1,37 +1,37 @@ { - "specs": "corpus-specs", + "specs": "ci-corpus/specs", "folders": { - "ci-corpus/tex": { - "root": ".", + "tex": { + "root": "ci-corpus", "suffixes": [ "tex" ], "built": 4, "documents": { - "ci-corpus/tex/sheet-1.tex": "built", - "ci-corpus/tex/sheet-2.tex": "built", - "ci-corpus/tex/sheet-3.tex": "built", - "ci-corpus/tex/solutions-2.tex": "built" + "tex/sheet-1.tex": "built", + "tex/sheet-2.tex": "built", + "tex/sheet-3.tex": "built", + "tex/solutions-2.tex": "built" } }, - "ci-corpus/docx": { - "root": ".", + "docx": { + "root": "ci-corpus", "suffixes": [ "docx" ], "built": 1, "documents": { - "ci-corpus/docx/sheet.docx": "built" + "docx/sheet.docx": "built" } }, - "ci-corpus/pdf": { - "root": ".", + "pdf": { + "root": "ci-corpus", "suffixes": [ "pdf" ], "built": 1, "documents": { - "ci-corpus/pdf/sheet-1.pdf": "built" + "pdf/sheet-1.pdf": "built" } } } diff --git a/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml b/ci-corpus/specs/docx/in2lambda-spec.yaml similarity index 100% rename from corpus-specs/ci-corpus/docx/in2lambda-spec.yaml rename to ci-corpus/specs/docx/in2lambda-spec.yaml diff --git a/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml b/ci-corpus/specs/pdf/in2lambda-spec.yaml similarity index 100% rename from corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml rename to ci-corpus/specs/pdf/in2lambda-spec.yaml diff --git a/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml b/ci-corpus/specs/tex/in2lambda-spec.yaml similarity index 100% rename from corpus-specs/ci-corpus/tex/in2lambda-spec.yaml rename to ci-corpus/specs/tex/in2lambda-spec.yaml diff --git a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml deleted file mode 100644 index 54afdc5..0000000 --- a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml +++ /dev/null @@ -1,6 +0,0 @@ -ignore: Para text~'^(STRESS ANALYSIS|Sheet |Note:)|Description automatically generated' -question: ListItem -part: Para text~'^[A-Z]' -solution: Para text~'^\[' -strip: ['^\d+\.\s+', '^\['] -layout: PartsOneSol diff --git a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml deleted file mode 100644 index c4bf288..0000000 --- a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml +++ /dev/null @@ -1,5 +0,0 @@ -ignore: Header -question: Para -solution: ListItem -strip: ['^\d+\.\s+'] -layout: PartsOneSol diff --git a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml deleted file mode 100644 index a3c7db3..0000000 --- a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml +++ /dev/null @@ -1,5 +0,0 @@ -ignore: Para text~'Figure [0-9]:' -question: Para label~'^(1a|Q[0-9])' -part: text~'.' -strip: ['^1a\) ', '^Q[0-9]\s*', '^- ', '^[0-9]+\. ', '^i+\)\s*'] -layout: PartsOneSol diff --git a/gate-baseline.json b/gate-baseline.json deleted file mode 100644 index 52bedac..0000000 --- a/gate-baseline.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "specs": "corpus-specs", - "folders": { - "UCL_MechEng": { - "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", - "suffixes": [ - "pdf" - ], - "built": 0, - "documents": { - "UCL_MechEng/Tutorial_2_Solutions.pdf": "skipped", - "UCL_MechEng/Worksheet_1.pdf": "faulted", - "UCL_MechEng/Worksheet_2.pdf": "faulted" - } - }, - "PHYS40002-Mechanics/problem_sheets_and_figures": { - "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", - "suffixes": [ - "tex" - ], - "built": 0, - "documents": { - "PHYS40002-Mechanics/problem_sheets_and_figures/figures/tunnel-potential.tex": "skipped", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS1.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS2.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS3.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS4.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS5.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS6.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS7.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS8.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS9.tex": "faulted" - } - }, - "MECH60014_Stress_analysis_3": { - "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", - "suffixes": [ - "docx" - ], - "built": 0, - "documents": { - "MECH60014_Stress_analysis_3/Stress_Sheet3_Sol(1).docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet4.docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet5 (2).docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet6 (1).docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet6_ Sol (1).docx": "faulted" - } - } - } -} diff --git a/tests/test_cli.py b/tests/test_cli.py index 77805be..cdd2362 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -144,10 +144,10 @@ def record(*args, **kwargs): def test_gate_defaults(): - args = build_parser().parse_args(["gate", "gate-baseline.json"]) + args = build_parser().parse_args(["gate", "corpus-specs/gate-baseline.json"]) assert args.command == "gate" - assert args.baseline == Path("gate-baseline.json") + assert args.baseline == Path("corpus-specs/gate-baseline.json") assert args.record is False assert args.cache == Path.home() / ".cache" / "in2lambda-agent" # Chosen when the command runs, so that two runs do not share a directory. diff --git a/tests/test_gate.py b/tests/test_gate.py index 6dd7921..afaf96d 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -219,16 +219,16 @@ def test_the_folder_line_names_each_document_that_did_worse(): counts={"faulted": 1}, recorded=0, regressions=[ - gate.Regression("UCL_MechEng/Worksheet_3.pdf", "built", "faulted", "KaTeX"), - gate.Regression("UCL_MechEng/Worksheet_4.pdf", "built", gate.MISSING), + gate.Regression("tex/sheet-3.tex", "built", "faulted", "KaTeX"), + gate.Regression("tex/sheet-4.tex", "built", gate.MISSING), ], ) - first, second, third = gate.folder_line("UCL_MechEng", summary).splitlines() + first, second, third = gate.folder_line("tex", summary).splitlines() - assert first.startswith("UCL_MechEng") - assert second == " worse UCL_MechEng/Worksheet_3.pdf built -> faulted: KaTeX" - assert third == " worse UCL_MechEng/Worksheet_4.pdf built -> missing" + assert first.startswith("tex") + assert second == " worse tex/sheet-3.tex built -> faulted: KaTeX" + assert third == " worse tex/sheet-4.tex built -> missing" def test_the_folder_line_says_where_no_count_is_recorded(baseline, tmp_path): @@ -280,27 +280,13 @@ def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): REPOSITORY = Path(__file__).resolve().parent.parent -@pytest.mark.parametrize( - ("committed", "named"), - [ - # The private corpus, which the workbench check replays, and the - # committed one, which the workflow replays. Two files, because a - # clone has the second and not the first. - ( - "gate-baseline.json", - { - "UCL_MechEng", - "PHYS40002-Mechanics/problem_sheets_and_figures", - "MECH60014_Stress_analysis_3", - }, - ), - ("ci-baseline.json", {"ci-corpus/tex", "ci-corpus/docx", "ci-corpus/pdf"}), - ], -) -def test_a_committed_baseline_names_its_folders_and_their_specs(committed, named): - baseline = gate.read_baseline(REPOSITORY / committed) +def test_the_committed_baseline_names_its_folders_and_their_specs(): + # The baseline for ExampleContents is not committed: its specs quote the + # headings of private documents and it names their files, so it is written + # beside them at corpus-specs/gate-baseline.json, which .gitignore covers. + baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") - assert set(baseline.folders) == named + assert set(baseline.folders) == {"tex", "docx", "pdf"} for name, folder in baseline.folders.items(): # The spec each folder replays, at the path the sweep reads it from. assert (REPOSITORY / baseline.specs / name / SPEC_NAME).is_file() @@ -309,8 +295,10 @@ def test_a_committed_baseline_names_its_folders_and_their_specs(committed, named assert folder.documents -def test_the_private_baseline_reads_the_corpus_where_it_is(): - baseline = gate.read_baseline(REPOSITORY / "gate-baseline.json") +def test_the_committed_baseline_names_no_path_outside_the_repository(): + # An absolute root is a path on one machine, and under ExampleContents it + # is also the name of a folder of private documents. + baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") - # The corpus is not in the repository and is read at its own path. - assert all(folder.root.is_absolute() for folder in baseline.folders.values()) + assert not baseline.specs.is_absolute() + assert all(not folder.root.is_absolute() for folder in baseline.folders.values()) From 0c558bada91a540f4277c0209b3fc46d71540877 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 02:34:59 +0100 Subject: [PATCH 4/6] implement: Gate every merge on a replay sweep over the corpus (t21) --- .github/workflows/gate.yml | 8 +-- .gitignore | 4 -- README.md | 42 ++++++++++++---- ci-baseline.json | 5 +- ci-corpus/specs/pdf/in2lambda-spec.yaml | 13 +++-- ci-corpus/specs/tex/in2lambda-spec.yaml | 12 +++-- ci-corpus/tex/sheet-1.tex | 40 ++++++--------- ci-corpus/tex/sheet-2-solutions.tex | 24 +++++++++ ci-corpus/tex/sheet-2.tex | 24 ++++----- ci-corpus/tex/sheet-3.tex | 42 ++++++---------- ci-corpus/tex/solutions-2.tex | 27 ---------- .../in2lambda-spec.yaml | 6 +++ .../in2lambda-spec.yaml | 5 ++ corpus-specs/UCL_MechEng/in2lambda-spec.yaml | 5 ++ gate-baseline.json | 50 +++++++++++++++++++ tests/test_cli.py | 4 +- tests/test_gate.py | 42 ++++++++++++++-- 17 files changed, 226 insertions(+), 127 deletions(-) create mode 100644 ci-corpus/tex/sheet-2-solutions.tex delete mode 100644 ci-corpus/tex/solutions-2.tex create mode 100644 corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml create mode 100644 corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml create mode 100644 corpus-specs/UCL_MechEng/in2lambda-spec.yaml create mode 100644 gate-baseline.json diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 1ff0647..cab70bd 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -1,9 +1,9 @@ # The merge gate. Every push to main and every pull request runs the tests and # then replays the three folders of ci-corpus, the committed corpus, which -# ci-baseline.json names. corpus-specs/gate-baseline.json names the folders of -# the private corpus, which a runner cannot read, and the workbench check -# replays those. This job reports on a branch; the workbench check is what -# holds the merge. +# ci-baseline.json names. This job is a required status check on main, so a +# branch whose job is red cannot be merged. gate-baseline.json names the +# folders of the private corpus, which a runner cannot read, and the workbench +# check replays those on the machine that holds it. name: gate on: diff --git a/.gitignore b/.gitignore index fdcb1ae..ea71284 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,6 @@ __pycache__/ dist/ # in2lambda's KaTeX converter writes this into the working directory on import. log -# The specs for ExampleContents, and the baseline recorded over it. Both quote -# private documents: a spec quotes their headings and the baseline names their -# files. The specs for the committed corpus are in ci-corpus/specs. -corpus-specs/ # The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex # in the workflow, so the repository holds the tex source and not the output. ci-corpus/pdf/*.pdf diff --git a/README.md b/README.md index efa58f8..e0a4591 100644 --- a/README.md +++ b/README.md @@ -318,14 +318,18 @@ There are two corpora and a baseline for each: | Baseline | Corpus | Run by | | --- | --- | --- | -| `corpus-specs/gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | +| `gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | | `ci-baseline.json` | the three folders of `ci-corpus` | `.github/workflows/gate.yml` | -`ci-corpus` is synthetic, so the repository holds the corpus, its specs in -`ci-corpus/specs/` and `ci-baseline.json`. `ExampleContents` is a set of private -documents, and a spec quotes the headings of one and a baseline names their files, so -`corpus-specs/` is in `.gitignore` and `corpus-specs/gate-baseline.json` is written -beside the specs it names. Write both before running the local gate. +Both baselines are committed, and so are both trees of specs: `ci-corpus/specs/` for +`ci-corpus` and `corpus-specs/` for `ExampleContents`. The gate runs in a worktree, and +a worktree holds what the repository holds. + +`ci-corpus` is synthetic, so the repository holds its documents as well. +`ExampleContents` is a set of private documents and is never in the repository: the +gate reads it at the absolute `root` that `gate-baseline.json` gives, which is a path +on the machine the check runs on. What the repository holds of that corpus is the +heading patterns its specs select on and the file names its baseline records. Each folder's `root` and `suffixes` are written by hand. `built`, the count of documents that built, and `documents`, the outcome of each single document, are what `--record` @@ -360,11 +364,27 @@ changed, or GitHub evicted the entry — Mathpix cannot be called and the `pdf` builds 0 against a recorded 1, so the job fails. Push that branch to a branch of this repository, where the secrets are read, and the job runs Mathpix once. -The `gate` job reports on a branch; it does not hold the merge. The workbench merges a -branch as soon as its own check passes, and its own check runs -`in2lambda-agent gate corpus-specs/gate-baseline.json` over `ExampleContents` after the -tests. `ExampleContents` is the larger corpus of the two, so the local gate is the -stricter check. +`gate` is a required status check on `main`, so `gh pr merge` refuses a branch whose +job is red, and the workbench, which merges through `gh`, refuses it too. This command +sets the requirement, and `gh` substitutes `{owner}` and `{repo}`: + +```sh +gh api -X PUT repos/{owner}/{repo}/branches/main/protection --input - <<'EOF' +{"required_status_checks":{"strict":false,"contexts":["gate"]}, + "enforce_admins":false,"required_pull_request_reviews":null,"restrictions":null} +EOF +``` + +This command reports what is required now: + +```sh +gh api repos/{owner}/{repo}/branches/main/protection/required_status_checks +``` + +The workbench check runs the local gate as well: `poetry install -q --with dev && +poetry run pytest -q && poetry run in2lambda-agent gate gate-baseline.json`. +`ExampleContents` is the larger corpus of the two, so the local gate reads more +documents than the job does. ## Docker diff --git a/ci-baseline.json b/ci-baseline.json index 2d0e4c8..3311388 100644 --- a/ci-baseline.json +++ b/ci-baseline.json @@ -6,12 +6,11 @@ "suffixes": [ "tex" ], - "built": 4, + "built": 3, "documents": { "tex/sheet-1.tex": "built", "tex/sheet-2.tex": "built", - "tex/sheet-3.tex": "built", - "tex/solutions-2.tex": "built" + "tex/sheet-3.tex": "built" } }, "docx": { diff --git a/ci-corpus/specs/pdf/in2lambda-spec.yaml b/ci-corpus/specs/pdf/in2lambda-spec.yaml index 2a668e6..3e4997f 100644 --- a/ci-corpus/specs/pdf/in2lambda-spec.yaml +++ b/ci-corpus/specs/pdf/in2lambda-spec.yaml @@ -1,5 +1,10 @@ +# The spec for the pdf set, which is sheet-1.tex compiled by xelatex and read +# back by Mathpix. The OCR writes the sheet's shape back: the question as a +# paragraph, its parts as a numbered list, and the solutions under the +# `Solutions` heading. The selectors are the tex set's for that reason. ignore: Header -question: ListItem text~'^[A-Z]' -solution: after Header text=Solutions, ListItem -strip: ['^\d+\. '] -layout: PartsOneSol +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/ci-corpus/specs/tex/in2lambda-spec.yaml b/ci-corpus/specs/tex/in2lambda-spec.yaml index 1754146..c95f041 100644 --- a/ci-corpus/specs/tex/in2lambda-spec.yaml +++ b/ci-corpus/specs/tex/in2lambda-spec.yaml @@ -1,4 +1,10 @@ +# The spec for the tex set: each question a paragraph, its parts the items of +# the list under it, and its solutions under a `Solutions` heading — in the +# same file for sheet-1 and sheet-3, and in sheet-2-solutions.tex for sheet-2. +# A solution begins with its label, `1(a)`, so no solution matches `question`. ignore: Header -question: ListItem -strip: ['^\d+\.\s+'] -layout: PartsOneSol +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/ci-corpus/tex/sheet-1.tex b/ci-corpus/tex/sheet-1.tex index 3127901..d62b66d 100644 --- a/ci-corpus/tex/sheet-1.tex +++ b/ci-corpus/tex/sheet-1.tex @@ -1,6 +1,6 @@ % Synthetic, in the shape of the corpus's problem sheets: a starred section -% heading, an enumerate of questions with lettered parts, and a solutions -% section at the end. Nothing here is copied from ExampleContents. +% heading, each question a paragraph with its parts as a lettered list, and a +% solutions section at the end. Nothing here is copied from ExampleContents. \documentclass[12pt]{article} \usepackage{amsmath} \usepackage{graphicx} @@ -9,38 +9,28 @@ \section*{Problem Sheet 1: Kinematics} -\begin{enumerate} +A ball is thrown straight up at $20\,\mathrm{m/s}$. -\item A ball is thrown straight up at $20\,\mathrm{m/s}$. - \begin{enumerate} - \item Find the greatest height it reaches. - \item Find its time of flight. - \end{enumerate} +\begin{enumerate} +\item Find the greatest height it reaches. +\item Find its time of flight. +\end{enumerate} -\item A block of mass $m$ rests on a slope of angle $\theta$. - \begin{enumerate} - \item Name the three forces acting on the block. - \item Find the least coefficient of friction that holds it still. - \end{enumerate} +A block of mass $m$ rests on a slope of angle $\theta$. +\begin{enumerate} +\item Name the three forces acting on the block. +\item Find the least coefficient of friction that holds it still. \end{enumerate} \section*{Solutions} -\begin{enumerate} +1(a) $h = v^2 / 2g = 20.4\,\mathrm{m}$ -\item - \begin{enumerate} - \item $h = v^2 / 2g = 20.4\,\mathrm{m}$ - \item $t = 2v/g = 4.08\,\mathrm{s}$ - \end{enumerate} +1(b) $t = 2v/g = 4.08\,\mathrm{s}$ -\item - \begin{enumerate} - \item Weight, the normal reaction, and friction along the slope. - \item $\mu = \tan\theta$ - \end{enumerate} +2(a) Weight, the normal reaction, and friction along the slope. -\end{enumerate} +2(b) $\mu = \tan\theta$ \end{document} diff --git a/ci-corpus/tex/sheet-2-solutions.tex b/ci-corpus/tex/sheet-2-solutions.tex new file mode 100644 index 0000000..9ba70bf --- /dev/null +++ b/ci-corpus/tex/sheet-2-solutions.tex @@ -0,0 +1,24 @@ +% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the +% shape that first failed a sweep. The name is what pairs the two: `pair` reads +% the stem before the `-solutions` ending, finds sheet-2.tex beside it, and the +% run freezes the questions first and these second. The spec reads every block +% under the `Solutions` heading as a solution, so this file adds no question. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Answers} + +\subsection*{Solutions} + +1(a) Take the divergence term by term; each pair cancels. + +1(b) $B = \mu_0 m / 2\pi z^3$ + +2(a) $r = mv / qB$ + +2(b) $T = 2\pi m / qB$ + +\end{document} diff --git a/ci-corpus/tex/sheet-2.tex b/ci-corpus/tex/sheet-2.tex index f18652d..aee9b53 100644 --- a/ci-corpus/tex/sheet-2.tex +++ b/ci-corpus/tex/sheet-2.tex @@ -1,6 +1,6 @@ % Synthetic: questions with lettered parts and no solutions on the sheet. The -% solutions are in solutions-2.tex, which is a document of its own, as the -% corpus's problem sheets are. Nothing here is copied from ExampleContents. +% solutions are in sheet-2-solutions.tex, which `pair` freezes as the draft's +% second source. Nothing here is copied from ExampleContents. \documentclass[12pt]{article} \usepackage{amsmath} \usepackage{graphicx} @@ -9,20 +9,18 @@ \section*{Problem Sheet 2: Fields} -\begin{enumerate} +A dipole sits at the origin. -\item A dipole sits at the origin. - \begin{enumerate} - \item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. - \item Find the field on the axis at a distance $z$. - \end{enumerate} +\begin{enumerate} +\item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. +\item Find the field on the axis at a distance $z$. +\end{enumerate} -\item A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. - \begin{enumerate} - \item Find the radius of its circular path. - \item Find the period of the motion. - \end{enumerate} +A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. +\begin{enumerate} +\item Find the radius of its circular path. +\item Find the period of the motion. \end{enumerate} \end{document} diff --git a/ci-corpus/tex/sheet-3.tex b/ci-corpus/tex/sheet-3.tex index 39d12d8..e19e16d 100644 --- a/ci-corpus/tex/sheet-3.tex +++ b/ci-corpus/tex/sheet-3.tex @@ -9,42 +9,30 @@ \section*{Problem Sheet 3: Statics} -\begin{enumerate} - -\item The beam below carries a load $W$ at its midpoint. +The beam below carries a load $W$ at its midpoint. +\includegraphics[width=0.2\textwidth]{figures/ball.png} - \includegraphics[width=0.2\textwidth]{figures/ball.png} - - \begin{enumerate} - \item Find the reaction at each support. - \item Find the bending moment at the midpoint. - \end{enumerate} +\begin{enumerate} +\item Find the reaction at each support. +\item Find the bending moment at the midpoint. +\end{enumerate} -\item A ladder of mass $m$ leans against a smooth wall. - \begin{enumerate} - \item Draw the forces acting on the ladder. - \item Find the least angle at which it does not slip. - \end{enumerate} +A ladder of mass $m$ leans against a smooth wall. +\begin{enumerate} +\item Draw the forces acting on the ladder. +\item Find the least angle at which it does not slip. \end{enumerate} \section*{Solutions} -\begin{enumerate} +1(a) $W/2$ at each support. -\item - \begin{enumerate} - \item $W/2$ at each support. - \item $M = WL/4$ - \end{enumerate} +1(b) $M = WL/4$ -\item - \begin{enumerate} - \item The weight at the centre, the normal reaction at the wall, and the - normal reaction and friction at the floor. - \item $\tan\alpha = 1/2\mu$ - \end{enumerate} +2(a) The weight at the centre, the normal reaction at the wall, and the normal +reaction and friction at the floor. -\end{enumerate} +2(b) $\tan\alpha = 1/2\mu$ \end{document} diff --git a/ci-corpus/tex/solutions-2.tex b/ci-corpus/tex/solutions-2.tex deleted file mode 100644 index 0cd053c..0000000 --- a/ci-corpus/tex/solutions-2.tex +++ /dev/null @@ -1,27 +0,0 @@ -% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the -% shape that first failed a sweep — a document with no questions in it. -\documentclass[12pt]{article} -\usepackage{amsmath} -\usepackage{graphicx} - -\begin{document} - -\section*{Problem Sheet 2: Solutions} - -\begin{enumerate} - -\item - \begin{enumerate} - \item Take the divergence term by term; each pair cancels. - \item $B = \mu_0 m / 2\pi z^3$ - \end{enumerate} - -\item - \begin{enumerate} - \item $r = mv / qB$ - \item $T = 2\pi m / qB$ - \end{enumerate} - -\end{enumerate} - -\end{document} diff --git a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml new file mode 100644 index 0000000..54afdc5 --- /dev/null +++ b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Para text~'^(STRESS ANALYSIS|Sheet |Note:)|Description automatically generated' +question: ListItem +part: Para text~'^[A-Z]' +solution: Para text~'^\[' +strip: ['^\d+\.\s+', '^\['] +layout: PartsOneSol diff --git a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml new file mode 100644 index 0000000..c4bf288 --- /dev/null +++ b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: Para +solution: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml new file mode 100644 index 0000000..a3c7db3 --- /dev/null +++ b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Para text~'Figure [0-9]:' +question: Para label~'^(1a|Q[0-9])' +part: text~'.' +strip: ['^1a\) ', '^Q[0-9]\s*', '^- ', '^[0-9]+\. ', '^i+\)\s*'] +layout: PartsOneSol diff --git a/gate-baseline.json b/gate-baseline.json new file mode 100644 index 0000000..52bedac --- /dev/null +++ b/gate-baseline.json @@ -0,0 +1,50 @@ +{ + "specs": "corpus-specs", + "folders": { + "UCL_MechEng": { + "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", + "suffixes": [ + "pdf" + ], + "built": 0, + "documents": { + "UCL_MechEng/Tutorial_2_Solutions.pdf": "skipped", + "UCL_MechEng/Worksheet_1.pdf": "faulted", + "UCL_MechEng/Worksheet_2.pdf": "faulted" + } + }, + "PHYS40002-Mechanics/problem_sheets_and_figures": { + "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", + "suffixes": [ + "tex" + ], + "built": 0, + "documents": { + "PHYS40002-Mechanics/problem_sheets_and_figures/figures/tunnel-potential.tex": "skipped", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS1.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS2.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS3.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS4.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS5.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS6.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS7.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS8.tex": "faulted", + "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS9.tex": "faulted" + } + }, + "MECH60014_Stress_analysis_3": { + "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", + "suffixes": [ + "docx" + ], + "built": 0, + "documents": { + "MECH60014_Stress_analysis_3/Stress_Sheet3_Sol(1).docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet4.docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet5 (2).docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet6 (1).docx": "faulted", + "MECH60014_Stress_analysis_3/Stress_Sheet6_ Sol (1).docx": "faulted" + } + } + } +} diff --git a/tests/test_cli.py b/tests/test_cli.py index cdd2362..77805be 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -144,10 +144,10 @@ def record(*args, **kwargs): def test_gate_defaults(): - args = build_parser().parse_args(["gate", "corpus-specs/gate-baseline.json"]) + args = build_parser().parse_args(["gate", "gate-baseline.json"]) assert args.command == "gate" - assert args.baseline == Path("corpus-specs/gate-baseline.json") + assert args.baseline == Path("gate-baseline.json") assert args.record is False assert args.cache == Path.home() / ".cache" / "in2lambda-agent" # Chosen when the command runs, so that two runs do not share a directory. diff --git a/tests/test_gate.py b/tests/test_gate.py index afaf96d..2a91867 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -6,7 +6,7 @@ from test_corpus import make_set from test_pipeline import SPEC, TEX_SPEC -from in2lambda_agent import gate +from in2lambda_agent import gate, pair from in2lambda_agent.gate import Baseline, Folder from in2lambda_agent.settings import Settings from in2lambda_agent.spec import SPEC_NAME @@ -281,9 +281,6 @@ def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): def test_the_committed_baseline_names_its_folders_and_their_specs(): - # The baseline for ExampleContents is not committed: its specs quote the - # headings of private documents and it names their files, so it is written - # beside them at corpus-specs/gate-baseline.json, which .gitignore covers. baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") assert set(baseline.folders) == {"tex", "docx", "pdf"} @@ -302,3 +299,40 @@ def test_the_committed_baseline_names_no_path_outside_the_repository(): assert not baseline.specs.is_absolute() assert all(not folder.root.is_absolute() for folder in baseline.folders.values()) + + +def test_the_local_baseline_and_the_specs_it_reads_are_both_committed(): + # The workbench check runs `gate gate-baseline.json` in a worktree of its + # own. Both files are read from the repository, so a worktree that holds + # neither fails the check in read_baseline before a document is swept. + baseline = gate.read_baseline(REPOSITORY / "gate-baseline.json") + + assert set(baseline.folders) == { + "UCL_MechEng", + "PHYS40002-Mechanics/problem_sheets_and_figures", + "MECH60014_Stress_analysis_3", + } + for name in baseline.folders: + assert (REPOSITORY / baseline.specs / name / SPEC_NAME).is_file() + + +def test_the_ci_corpus_pairs_a_solutions_document_with_its_questions(): + # The corpus exists to exercise the separate-solutions document, which it + # does only when `pair` matches the file's name. Name it so that it does + # not — solutions-2.tex rather than sheet-2-solutions.tex — and the sweep + # reads it as a sheet of its own and the path is never run. + assert pair.solutions_beside(REPOSITORY / "ci-corpus/tex/sheet-2.tex") is not None + + +def test_no_document_of_the_ci_corpus_is_a_solutions_file(): + # A solutions document is frozen as the second source of the questions + # document beside it, so a sweep gives it no row of its own. + baseline = gate.read_baseline(REPOSITORY / "ci-baseline.json") + + named = [ + document + for folder in baseline.folders.values() + for document in folder.documents + if pair.questions_stem(Path(document)) is not None + ] + assert named == [] From 17b8c22de691f90cb2ad4ac9962d55c6713cb803 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 03:05:46 +0100 Subject: [PATCH 5/6] implement: Gate every merge on a replay sweep over the corpus (t21) --- .github/workflows/gate.yml | 15 +++++++++++++-- README.md | 16 +++++++++++++++- tests/test_docs.py | 23 +++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index cab70bd..cefe0a2 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -20,13 +20,24 @@ jobs: steps: - uses: actions/checkout@v4 + # Pandoc reads every source document, so its version decides the blocks + # a spec selects. ubuntu-24.04 packages pandoc 3.1.3, under which every + # ci-corpus document faults on a block no selector reaches, and the + # tests fail with it. This release is the one the baselines were + # recorded under. + - name: Install pandoc + run: | + curl -fsSL -o "${RUNNER_TEMP}/pandoc.deb" \ + https://github.com/jgm/pandoc/releases/download/3.9.0.2/pandoc-3.9.0.2-1-amd64.deb + sudo dpkg -i "${RUNNER_TEMP}/pandoc.deb" + pandoc --version | head -1 + # The Dockerfile's package list, which is the one record of what the PDF # generator's template.latex loads. - - name: Install pandoc, poppler and TeX Live + - name: Install poppler and TeX Live run: | sudo apt-get update sudo apt-get install --no-install-recommends -y \ - pandoc \ poppler-utils \ texlive-xetex \ texlive-latex-recommended \ diff --git a/README.md b/README.md index e0a4591..8221fa4 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,21 @@ Both baselines are committed, and so are both trees of specs: `ci-corpus/specs/` `ci-corpus` and `corpus-specs/` for `ExampleContents`. The gate runs in a worktree, and a worktree holds what the repository holds. -`ci-corpus` is synthetic, so the repository holds its documents as well. +`ci-corpus` is synthetic, so the repository holds its documents — every one but the +PDF, which xelatex compiles from `ci-corpus/tex/sheet-1.tex`. Run the command the +workflow runs before `gate ci-baseline.json`, because the PDF's bytes are the key the +OCR cache reads under: + +```sh +cd ci-corpus/tex +SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ + xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex +rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log +``` + +Without the PDF the `pdf` folder holds no document, builds 0 against a recorded 1, +and the gate exits 1. + `ExampleContents` is a set of private documents and is never in the repository: the gate reads it at the absolute `root` that `gate-baseline.json` gives, which is a path on the machine the check runs on. What the repository holds of that corpus is the diff --git a/tests/test_docs.py b/tests/test_docs.py index 35b43e0..bb85452 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -13,6 +13,7 @@ ROOT = Path(__file__).resolve().parent.parent README = (ROOT / "README.md").read_text(encoding="utf-8") HOW_IT_WORKS = (ROOT / "docs" / "how-it-works.md").read_text(encoding="utf-8") +WORKFLOW = (ROOT / ".github" / "workflows" / "gate.yml").read_text(encoding="utf-8") def _options(parser: argparse.ArgumentParser) -> set[str]: @@ -60,3 +61,25 @@ def test_how_it_works_names_every_stage(): assert len(names) == 9 missing = [one for one in sorted(names) if f"`{one}`" not in HOW_IT_WORKS] assert not missing + + +def test_the_readme_compiles_the_ci_corpus_pdf_as_the_workflow_does(): + # The repository does not hold ci-corpus/pdf, so the reader compiles the + # PDF before the gate reads it. The README and the workflow give the same + # command, because another command writes other bytes, and the PDF's bytes + # are the key the OCR cache reads under. + for line in ( + "SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1", + "xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex", + ): + assert line in WORKFLOW + assert line in README + + +def test_the_workflow_installs_a_pandoc_of_its_own(): + # ubuntu-24.04 packages pandoc 3.1.3, under which every ci-corpus document + # faults on a block no selector reaches. The job installs a pinned + # release, the one the baselines were recorded under. + apt = WORKFLOW.split("apt-get install", 1)[1].split("\n\n", 1)[0] + assert "pandoc" not in apt + assert "https://github.com/jgm/pandoc/releases/download/" in WORKFLOW From 634c8e8774800ed5a840caa0b0d1374214362c60 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 04:07:27 +0100 Subject: [PATCH 6/6] implement: Finish the gate: keep private specs and baselines out of the repository (t31) --- .github/workflows/gate.yml | 19 ++--- .gitignore | 4 ++ README.md | 58 +++++++-------- .../in2lambda-spec.yaml | 6 -- .../in2lambda-spec.yaml | 5 -- corpus-specs/UCL_MechEng/in2lambda-spec.yaml | 5 -- gate-baseline.json | 50 ------------- in2lambda_agent/gate.py | 49 ++++++++----- tests/test_docs.py | 22 ++++++ tests/test_gate.py | 72 +++++++++++++++---- 10 files changed, 154 insertions(+), 136 deletions(-) delete mode 100644 corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml delete mode 100644 corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml delete mode 100644 corpus-specs/UCL_MechEng/in2lambda-spec.yaml delete mode 100644 gate-baseline.json diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index cefe0a2..6b23bf6 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -1,9 +1,9 @@ # The merge gate. Every push to main and every pull request runs the tests and # then replays the three folders of ci-corpus, the committed corpus, which -# ci-baseline.json names. This job is a required status check on main, so a -# branch whose job is red cannot be merged. gate-baseline.json names the -# folders of the private corpus, which a runner cannot read, and the workbench -# check replays those on the machine that holds it. +# ci-baseline.json names. The job is CI's report on a branch and nothing +# merges on it: the workbench merges with `gh pr merge` as soon as its own +# check passes, and that check replays the private corpus, whose folders a +# runner cannot read, on the machine that holds it. name: gate on: @@ -95,19 +95,20 @@ jobs: run: poetry run in2lambda-agent gate ci-baseline.json # What this run would record, so that a baseline change is committed from - # what CI saw. Written whether or not the gate passed. + # what CI saw. Written whether or not the gate passed. The file is + # recorded where it is checked out, because a baseline names its specs + # relative to its own directory and a copy under RUNNER_TEMP would find + # none; the job throws the checkout away. - name: Record what this run did if: always() env: MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} - run: | - cp ci-baseline.json "${RUNNER_TEMP}/ci-baseline.json" - poetry run in2lambda-agent gate --record "${RUNNER_TEMP}/ci-baseline.json" + run: poetry run in2lambda-agent gate --record ci-baseline.json - name: Upload it if: always() uses: actions/upload-artifact@v4 with: name: recorded-baseline - path: ${{ runner.temp }}/ci-baseline.json + path: ci-baseline.json diff --git a/.gitignore b/.gitignore index ea71284..8a51ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,10 @@ __pycache__/ *.egg-info/ .venv/ dist/ +# The specs for ExampleContents quote the headings of private documents, and +# gate-baseline.json beside them records those documents' file names and the +# absolute path of the corpus on one machine. +corpus-specs/ # in2lambda's KaTeX converter writes this into the working directory on import. log # The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex diff --git a/README.md b/README.md index 8221fa4..0d85aa1 100644 --- a/README.md +++ b/README.md @@ -285,9 +285,9 @@ review, rejections where its value comes from. One document that fails is one row and not the end of the sweep, and a set whose folder cannot be copied is a row for each of its documents. -`--cache` (default `./.in2lambda-agent`, the directory `run` caches into) is where the -OCR of each PDF is kept. A sweep pointed at a cache that an earlier run filled makes no -Mathpix call, and needs no Mathpix credentials. +`--cache` is where the OCR of each PDF is kept. It defaults to `./.in2lambda-agent`, +the directory `run` caches into, so a sweep over PDFs that `run` has already converted +makes no Mathpix call and needs no Mathpix credentials. ## Gate @@ -311,19 +311,23 @@ tex built 2 faulted 0 build refused 0 skipped 0 no spec 2 worse tex/sheet-3.tex built -> no spec: replay: no model call is allowed ``` -Run the command from the repository root. The gate reads `specs`, and a folder's `root` -where `root` is relative, from the directory the command runs in. +The gate reads `specs`, and a folder's `root` where `root` is relative, from the +directory `BASELINE` is in, so the command gives the same run from any directory. There are two corpora and a baseline for each: | Baseline | Corpus | Run by | | --- | --- | --- | -| `gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | +| `corpus-specs/gate-baseline.json` | the three folders of `ExampleContents` | the workbench check | | `ci-baseline.json` | the three folders of `ci-corpus` | `.github/workflows/gate.yml` | -Both baselines are committed, and so are both trees of specs: `ci-corpus/specs/` for -`ci-corpus` and `corpus-specs/` for `ExampleContents`. The gate runs in a worktree, and -a worktree holds what the repository holds. +The repository holds `ci-baseline.json` and the specs it names, under +`ci-corpus/specs/`, because `ci-corpus` is synthetic. The repository holds neither the +specs for `ExampleContents` nor the baseline that names them: the specs quote the +headings of private documents, and the baseline records those documents' file names and +the path of the corpus on one machine. `.gitignore` lists `corpus-specs/`, and +`gate-baseline.json` sits in that directory beside the specs it reads, with `"specs": +"."`. `ci-corpus` is synthetic, so the repository holds its documents — every one but the PDF, which xelatex compiles from `ci-corpus/tex/sheet-1.tex`. Run the command the @@ -342,8 +346,7 @@ and the gate exits 1. `ExampleContents` is a set of private documents and is never in the repository: the gate reads it at the absolute `root` that `gate-baseline.json` gives, which is a path -on the machine the check runs on. What the repository holds of that corpus is the -heading patterns its specs select on and the file names its baseline records. +on the machine the check runs on. Each folder's `root` and `suffixes` are written by hand. `built`, the count of documents that built, and `documents`, the outcome of each single document, are what `--record` @@ -356,10 +359,10 @@ records 0 built for all three folders: pandoc's line wrapping is reported as a m delimiter error, and each document needs a fixing round that a replay does not run. The recorded outcomes are what the gate defends until a later ticket raises the count. -`--cache` defaults to `~/.cache/in2lambda-agent`, and not to the `./.in2lambda-agent` -that `run` and `corpus` cache into, because the gate runs in a worktree of its own: a -PDF converted on one branch is converted again on the next if the cache sits in the -branch's directory. `--work` defaults to a new directory under the system temp +`--cache` defaults to `~/.cache/in2lambda-agent`, which is outside every worktree, +because the gate runs in a worktree of its own: a PDF converted on one branch is +converted again on the next if the cache sits in the branch's directory. `--work` +defaults to a new directory under the system temp directory, which the gate does not delete: read the drafts of a folder that failed there. The gate also copies the spec tree into the work directory and replays the copy, because a sweep appends a record of each run beside the @@ -378,27 +381,20 @@ changed, or GitHub evicted the entry — Mathpix cannot be called and the `pdf` builds 0 against a recorded 1, so the job fails. Push that branch to a branch of this repository, where the secrets are read, and the job runs Mathpix once. -`gate` is a required status check on `main`, so `gh pr merge` refuses a branch whose -job is red, and the workbench, which merges through `gh`, refuses it too. This command -sets the requirement, and `gh` substitutes `{owner}` and `{repo}`: +The job is CI's report on a branch and no merge waits for it. The workbench merges with +`gh pr merge` as soon as its own check passes, and `gh pr merge` cannot wait for a +GitHub check, so requiring the job on `main` would refuse every merge the workbench +makes. -```sh -gh api -X PUT repos/{owner}/{repo}/branches/main/protection --input - <<'EOF' -{"required_status_checks":{"strict":false,"contexts":["gate"]}, - "enforce_admins":false,"required_pull_request_reviews":null,"restrictions":null} -EOF -``` - -This command reports what is required now: +The workbench check runs the gate over `ExampleContents`, which is the larger corpus of +the two: ```sh -gh api repos/{owner}/{repo}/branches/main/protection/required_status_checks +poetry install -q --with dev && poetry run pytest -q && poetry run in2lambda-agent gate /Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/corpus-specs/gate-baseline.json ``` -The workbench check runs the local gate as well: `poetry install -q --with dev && -poetry run pytest -q && poetry run in2lambda-agent gate gate-baseline.json`. -`ExampleContents` is the larger corpus of the two, so the local gate reads more -documents than the job does. +The path is absolute because the check runs in a worktree and the worktree holds +neither the baseline nor the specs it names. ## Docker diff --git a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml deleted file mode 100644 index 54afdc5..0000000 --- a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml +++ /dev/null @@ -1,6 +0,0 @@ -ignore: Para text~'^(STRESS ANALYSIS|Sheet |Note:)|Description automatically generated' -question: ListItem -part: Para text~'^[A-Z]' -solution: Para text~'^\[' -strip: ['^\d+\.\s+', '^\['] -layout: PartsOneSol diff --git a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml deleted file mode 100644 index c4bf288..0000000 --- a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml +++ /dev/null @@ -1,5 +0,0 @@ -ignore: Header -question: Para -solution: ListItem -strip: ['^\d+\.\s+'] -layout: PartsOneSol diff --git a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml deleted file mode 100644 index a3c7db3..0000000 --- a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml +++ /dev/null @@ -1,5 +0,0 @@ -ignore: Para text~'Figure [0-9]:' -question: Para label~'^(1a|Q[0-9])' -part: text~'.' -strip: ['^1a\) ', '^Q[0-9]\s*', '^- ', '^[0-9]+\. ', '^i+\)\s*'] -layout: PartsOneSol diff --git a/gate-baseline.json b/gate-baseline.json deleted file mode 100644 index 52bedac..0000000 --- a/gate-baseline.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "specs": "corpus-specs", - "folders": { - "UCL_MechEng": { - "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", - "suffixes": [ - "pdf" - ], - "built": 0, - "documents": { - "UCL_MechEng/Tutorial_2_Solutions.pdf": "skipped", - "UCL_MechEng/Worksheet_1.pdf": "faulted", - "UCL_MechEng/Worksheet_2.pdf": "faulted" - } - }, - "PHYS40002-Mechanics/problem_sheets_and_figures": { - "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", - "suffixes": [ - "tex" - ], - "built": 0, - "documents": { - "PHYS40002-Mechanics/problem_sheets_and_figures/figures/tunnel-potential.tex": "skipped", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS1.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS2.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS3.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS4.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS5.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS6.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS7.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS8.tex": "faulted", - "PHYS40002-Mechanics/problem_sheets_and_figures/mechanics_23-24_PS9.tex": "faulted" - } - }, - "MECH60014_Stress_analysis_3": { - "root": "/Users/peterbjohnson/code/lambdafeedback/in2lambda-agent/ExampleContents", - "suffixes": [ - "docx" - ], - "built": 0, - "documents": { - "MECH60014_Stress_analysis_3/Stress_Sheet3_Sol(1).docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet4.docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet5 (2).docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet6 (1).docx": "faulted", - "MECH60014_Stress_analysis_3/Stress_Sheet6_ Sol (1).docx": "faulted" - } - } - } -} diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py index 35ef428..e55e30e 100644 --- a/in2lambda_agent/gate.py +++ b/in2lambda_agent/gate.py @@ -13,9 +13,14 @@ check is what gives a baseline of no builds at all teeth: a folder where every document faults still notices the day one of them stops being read. A folder the baseline records no count for passes on any count, which is how a folder is -added to the gate before it replays to a build worth defending. The baseline is -committed, and a change to it belongs in a pull request that states why the -counts changed. +added to the gate before it replays to a build worth defending. A change to a +recorded count belongs in a pull request that states why the count changed. + +A baseline names its specs, and a corpus of its own, relative to the directory +the file is in. `ci-baseline.json` is at the root of the repository and names +`ci-corpus/specs`; the baseline for the private corpus is at +`corpus-specs/gate-baseline.json`, beside the specs it names, and the +repository holds neither file. """ import json @@ -65,9 +70,9 @@ class Folder: """One folder of a corpus, as the baseline holds it. Attributes: - root: The corpus directory the folder is under. The repository itself - for `ci-corpus`, which is committed; an absolute path for a corpus - outside the repository. + root: The corpus directory the folder is under, read relative to the + directory the baseline file is in. `ci-corpus` for the committed + corpus; an absolute path for a corpus outside the repository. suffixes: The file suffixes that are documents in the folder: `pdf` for a folder of scans, `tex` or `docx` for sources. built: How many documents built when the baseline was recorded, or None @@ -103,16 +108,22 @@ class Regression: @dataclass class Baseline: - """The committed file the gate compares a sweep against. + """The file the gate compares a sweep against. Attributes: - specs: The tree the folders' specs are kept in, relative to the - repository. Folder `A/B` reads `/A/B/in2lambda-spec.yaml`. + specs: The tree the folders' specs are kept in, read relative to the + directory the baseline file is in. Folder `A/B` reads + `/A/B/in2lambda-spec.yaml`. folders: The folders to run, by their path under their own root. + directory: The directory the baseline file was read from. `specs` and + a relative `root` are read from there, so the command gives the + same run whichever directory it is run in. The paths themselves are + held as they are written, so `--record` writes them back unchanged. """ specs: Path folders: dict[str, Folder] + directory: Path = Path(".") @dataclass @@ -153,16 +164,18 @@ def failed(self) -> bool: def read_baseline(path: Path) -> Baseline: - """Reads the committed baseline. + """Reads a baseline file. Args: - path: The JSON file. + path: The JSON file. The paths it names are read from the directory it + is in. Returns: The baseline. """ written = json.loads(Path(path).read_text(encoding="utf-8")) return Baseline( + directory=Path(path).parent, specs=Path(written["specs"]), folders={ name: Folder( @@ -212,8 +225,9 @@ def run( swept into its own directory under `work`, and the table is written there. Args: - baseline: The folders to run and what to compare against. In record - mode this run's counts and outcomes replace them. + baseline: The folders to run and what to compare against. Its `specs` + and each relative `root` are read from the directory it was read + from. In record mode this run's counts and outcomes replace them. record: Take this run as the new baseline rather than checking it. cache: Where the OCR of each PDF is kept. The directory is shared between worktrees, so Mathpix converts each PDF once. @@ -232,12 +246,15 @@ def run( # Fresh each run, so that a spec taken out of the tree is gone from the # copy the sweep reads rather than left over from the run before. shutil.rmtree(specs, ignore_errors=True) - if Path(baseline.specs).is_dir(): - shutil.copytree(baseline.specs, specs) + # Joining an absolute path to the baseline's directory returns the absolute + # path, so a private corpus named by its path on one machine is unchanged. + written = baseline.directory / baseline.specs + if written.is_dir(): + shutil.copytree(written, specs) report = Report() for name, folder in baseline.folders.items(): rows = corpus.sweep( - folder.root, + baseline.directory / folder.root, paths=[Path(name)], suffixes=folder.suffixes, results=work / name / "results.csv", diff --git a/tests/test_docs.py b/tests/test_docs.py index bb85452..467318a 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -76,6 +76,28 @@ def test_the_readme_compiles_the_ci_corpus_pdf_as_the_workflow_does(): assert line in README +def test_no_github_check_is_required_on_main(): + # The workbench merges with `gh pr merge` as soon as its own check passes, + # and `gh pr merge` cannot wait for a GitHub check, so a required check + # refuses every merge the workbench makes. The workflow is CI's report. + for text in (README, WORKFLOW): + assert "required_status_checks" not in text + assert "branch-protection" not in text + + +def test_the_readme_names_the_private_baseline_by_an_absolute_path(): + # The workbench check runs in a worktree, and the worktree holds neither + # the baseline for ExampleContents nor the specs it names. + check = next( + line + for line in README.splitlines() + if "in2lambda-agent gate " in line and "gate-baseline.json" in line + ) + path = check.split("in2lambda-agent gate ", 1)[1].split()[0] + assert path.startswith("/") + assert path.endswith("/corpus-specs/gate-baseline.json") + + def test_the_workflow_installs_a_pandoc_of_its_own(): # ubuntu-24.04 packages pandoc 3.1.3, under which every ci-corpus document # faults on a block no selector reaches. The job installs a pinned diff --git a/tests/test_gate.py b/tests/test_gate.py index 2a91867..7e2c2ca 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -1,5 +1,6 @@ """The merge gate: a replay over a corpus, checked against a recorded baseline.""" +import json from pathlib import Path import pytest @@ -245,7 +246,56 @@ def test_the_baseline_survives_being_written_and_read(baseline, tmp_path): gate.write_baseline(baseline, path) read = gate.read_baseline(path) - assert read == baseline + assert (read.specs, read.folders) == (baseline.specs, baseline.folders) + # The directory the paths are read from is the file's own and is not a + # field of the file. + assert read.directory == tmp_path + + +@pytest.fixture +def beside(tmp_path): + """A baseline naming its specs and its corpus relative to its own directory.""" + made = tmp_path / "beside" + (made / "sheets").mkdir(parents=True) + (made / "sheets" / SPEC_NAME).write_text(SPEC) + make_set(made / "corpus", "sheets", ["sheet.md"]) + (made / "baseline.json").write_text( + json.dumps( + { + "specs": ".", + "folders": {"sheets": {"root": "corpus", "suffixes": ["md"]}}, + } + ) + ) + return made / "baseline.json" + + +def test_the_specs_and_a_relative_root_are_read_beside_the_baseline( + beside, tmp_path, monkeypatch +): + # The private baseline sits beside the specs it names, outside the + # repository, and the workbench check runs the gate in a worktree. Reading + # the file's paths from its own directory gives the same run from any + # directory. + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + + report = run(gate.read_baseline(beside), tmp_path) + + assert report.folders["sheets"].built == 1 + + +def test_recording_writes_the_paths_as_they_were_written(beside, tmp_path): + read = gate.read_baseline(beside) + + run(read, tmp_path, record=True) + gate.write_baseline(read, beside) + + written = json.loads(beside.read_text()) + assert written["specs"] == "." + assert written["folders"]["sheets"]["root"] == "corpus" + assert written["folders"]["sheets"]["built"] == 1 def test_a_run_writes_nothing_under_the_directory_it_was_run_from( @@ -301,19 +351,13 @@ def test_the_committed_baseline_names_no_path_outside_the_repository(): assert all(not folder.root.is_absolute() for folder in baseline.folders.values()) -def test_the_local_baseline_and_the_specs_it_reads_are_both_committed(): - # The workbench check runs `gate gate-baseline.json` in a worktree of its - # own. Both files are read from the repository, so a worktree that holds - # neither fails the check in read_baseline before a document is swept. - baseline = gate.read_baseline(REPOSITORY / "gate-baseline.json") - - assert set(baseline.folders) == { - "UCL_MechEng", - "PHYS40002-Mechanics/problem_sheets_and_figures", - "MECH60014_Stress_analysis_3", - } - for name in baseline.folders: - assert (REPOSITORY / baseline.specs / name / SPEC_NAME).is_file() +def test_the_private_corpus_keeps_its_specs_and_its_baseline_out_of_the_repository(): + # The specs for ExampleContents quote the headings of private documents, + # and the baseline beside them records those documents' file names and the + # path of the corpus on one machine. The workbench check names that + # baseline by its absolute path instead of reading it from the worktree. + assert "corpus-specs/" in (REPOSITORY / ".gitignore").read_text() + assert not (REPOSITORY / "gate-baseline.json").exists() def test_the_ci_corpus_pairs_a_solutions_document_with_its_questions():