From 93610dd036a4beb11d70f577904b9ec42ba811a0 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 00:41:11 +0100 Subject: [PATCH 1/2] implement: Gate every merge on a replay sweep over the corpus (t21) --- .github/branch-protection.json | 9 + .github/workflows/gate.yml | 100 ++++++++ .gitignore | 6 + README.md | 63 ++++- ci-corpus/docx/sheet.docx | Bin 0 -> 11101 bytes ci-corpus/pdf/.gitkeep | 0 ci-corpus/tex/figures/ball.png | Bin 0 -> 70 bytes ci-corpus/tex/sheet-1.tex | 46 ++++ ci-corpus/tex/sheet-2.tex | 28 +++ ci-corpus/tex/sheet-3.tex | 50 ++++ ci-corpus/tex/solutions-2.tex | 27 +++ .../in2lambda-spec.yaml | 6 + .../in2lambda-spec.yaml | 5 + corpus-specs/UCL_MechEng/in2lambda-spec.yaml | 5 + corpus-specs/baseline.json | 26 +++ .../ci-corpus/docx/in2lambda-spec.yaml | 6 + .../ci-corpus/pdf/in2lambda-spec.yaml | 5 + .../ci-corpus/tex/in2lambda-spec.yaml | 4 + in2lambda_agent/cli.py | 59 ++++- in2lambda_agent/corpus.py | 7 + in2lambda_agent/gate.py | 219 ++++++++++++++++++ in2lambda_agent/ocr.py | 27 ++- in2lambda_agent/pipeline.py | 14 +- tests/test_cli.py | 103 +++++++- tests/test_corpus.py | 16 ++ tests/test_gate.py | 198 ++++++++++++++++ tests/test_ocr.py | 15 +- tests/test_pipeline.py | 21 ++ 28 files changed, 1054 insertions(+), 11 deletions(-) create mode 100644 .github/branch-protection.json create mode 100644 .github/workflows/gate.yml create mode 100644 ci-corpus/docx/sheet.docx create mode 100644 ci-corpus/pdf/.gitkeep create mode 100644 ci-corpus/tex/figures/ball.png create mode 100644 ci-corpus/tex/sheet-1.tex create mode 100644 ci-corpus/tex/sheet-2.tex create mode 100644 ci-corpus/tex/sheet-3.tex create mode 100644 ci-corpus/tex/solutions-2.tex create mode 100644 corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml create mode 100644 corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml create mode 100644 corpus-specs/UCL_MechEng/in2lambda-spec.yaml create mode 100644 corpus-specs/baseline.json create mode 100644 corpus-specs/ci-corpus/docx/in2lambda-spec.yaml create mode 100644 corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml create mode 100644 corpus-specs/ci-corpus/tex/in2lambda-spec.yaml create mode 100644 in2lambda_agent/gate.py create mode 100644 tests/test_gate.py diff --git a/.github/branch-protection.json b/.github/branch-protection.json new file mode 100644 index 0000000..2d1362c --- /dev/null +++ b/.github/branch-protection.json @@ -0,0 +1,9 @@ +{ + "required_status_checks": { + "strict": false, + "contexts": ["gate"] + }, + "enforce_admins": false, + "required_pull_request_reviews": null, + "restrictions": null +} diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml new file mode 100644 index 0000000..9a5e845 --- /dev/null +++ b/.github/workflows/gate.yml @@ -0,0 +1,100 @@ +# The merge gate. Every push to main and every pull request runs the tests and +# then replays the folders corpus-specs/baseline.json names, which today are +# the three of ci-corpus, the committed corpus. The job is a required status +# check on main, so a branch that breaks a build cannot be merged. +name: gate + +on: + push: + branches: [main] + pull_request: + +jobs: + gate: + name: gate + # Pinned rather than ubuntu-latest: the runner's TeX Live decides what + # xelatex writes, and the PDF's bytes are the OCR cache's key. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + # The Dockerfile's package list, which is the one record of what the PDF + # generator's template.latex loads. + - name: Install pandoc, poppler and TeX Live + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + pandoc \ + poppler-utils \ + texlive-xetex \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-science \ + texlive-lang-chinese \ + texlive-lang-arabic \ + texlive-bibtex-extra \ + texlive-plain-generic \ + texlive-fonts-recommended \ + lmodern \ + fonts-noto-core \ + fonts-noto-cjk + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install poetry + run: pipx install poetry + + - name: Install the package + run: poetry install --with dev + + # SOURCE_DATE_EPOCH fixes the timestamp xelatex writes into the PDF, so + # the same tex source compiles to the same bytes and the OCR cache key + # does not change between runs. + - name: Compile the CI corpus PDF + run: | + cd ci-corpus/tex + SOURCE_DATE_EPOCH=0 FORCE_SOURCE_DATE=1 \ + xelatex -interaction=nonstopmode -output-directory=../pdf sheet-1.tex + rm -f ../pdf/sheet-1.aux ../pdf/sheet-1.log + + # Keyed by the PDF's bytes, which is also how ocr_pdf names its entry + # inside the directory. A restored entry for another PDF is unused. + - name: Restore the OCR cache + uses: actions/cache@v4 + with: + path: ~/.cache/in2lambda-agent + key: ocr-${{ hashFiles('ci-corpus/pdf/*.pdf') }} + restore-keys: ocr- + + - name: Tests + run: poetry run pytest -q + + - name: Gate + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: poetry run in2lambda-agent gate corpus-specs/baseline.json + + # What this run would record, so that a baseline change is committed from + # what CI saw. Written whether or not the gate passed. + - name: Record what this run did + if: always() + env: + MATHPIX_APP_ID: ${{ secrets.MATHPIX_APP_ID }} + MATHPIX_API_KEY: ${{ secrets.MATHPIX_API_KEY }} + run: | + cp corpus-specs/baseline.json "${RUNNER_TEMP}/baseline.json" + poetry run in2lambda-agent gate --record "${RUNNER_TEMP}/baseline.json" + + - name: Upload it + if: always() + uses: actions/upload-artifact@v4 + with: + name: recorded-baseline + path: ${{ runner.temp }}/baseline.json diff --git a/.gitignore b/.gitignore index 45e2c81..b7e303d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,9 @@ __pycache__/ dist/ # in2lambda's KaTeX converter writes this into the working directory on import. log +# The gate's CI corpus: xelatex compiles this PDF from ci-corpus/tex/sheet-1.tex +# in the workflow, so the repository holds the tex source and not the output. +ci-corpus/pdf/*.pdf +# Every run appends a line to the record beside the spec it read. The spec is +# what the gate replays; the record is one run's own and is not committed. +corpus-specs/**/in2lambda-agent-runs.jsonl diff --git a/README.md b/README.md index a3136a4..311908c 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ poetry run in2lambda-agent corpus ExampleContents --suffix tex --suffix md In full: ```sh -poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] +poetry run in2lambda-agent corpus ROOT [PATH ...] [--suffix S] [--replay] [--rounds N] [--results FILE] [--work DIR] [--specs DIR] [--cache DIR] ``` `ROOT` is the corpus directory and each `PATH` a folder under it to run, defaulting to @@ -209,6 +209,67 @@ how far the spec got alone — which is why a `built` row can still report block unassigned. `rejections` is always 0 while `--review` acts on nothing, and is the column a later review mode fills. +`--cache` (default `./.in2lambda-agent`) is where the OCR of each PDF is kept. A sweep +pointed at a cache that an earlier run filled makes no Mathpix call, and needs no +Mathpix credentials. + +## Gate + +Nothing merges without a replay over real documents. `gate` reruns the saved specs +over the folders a baseline file names, and compares how many documents each folder +built with the count the baseline records: + +```sh +poetry run in2lambda-agent gate corpus-specs/baseline.json [--record] [--cache DIR] [--work DIR] +``` + +Every run is `corpus --replay`, so no model call is made. The command prints one line +per folder, saying what the folder did beside what the baseline records, and exits 1 +when a folder builds fewer documents than the recorded count: + +``` +work /tmp/in2lambda-agent-gate-3f1a +ci-corpus/tex built 0 faulted 4 build refused 0 skipped 0 (baseline built 4) +ci-corpus/docx built 1 faulted 0 build refused 0 skipped 0 (baseline built 1) +ci-corpus/pdf built 1 faulted 0 build refused 0 skipped 0 (baseline built 1) +``` + +Run the command from the repository root: `specs`, and a folder's `root` where it is +relative, are read from there. + +`corpus-specs/baseline.json` is committed. Each folder's `root` and `suffixes` are +written by hand; `built` is what `--record` writes. A folder the file records no +`built` for passes on any count, and its line reads `(not recorded)`, which is how a +folder joins the gate before its specs replay to a build worth defending. A change to +a recorded count belongs in a pull request that says why the count changed. + +Today the file names the three folders of `ci-corpus`, the committed corpus: three tex +sheets, one docx, and one PDF that the workflow compiles from `ci-corpus/tex/sheet-1.tex` +with xelatex. The three folders of `ExampleContents`, the private corpus, have their +specs under `corpus-specs/` and are not in the file yet: every document there replays +to `faulted`, because pandoc's line wrapping is reported as a math delimiter error and +each one needs a fixing round that a replay does not run. + +`--cache` defaults to `~/.cache/in2lambda-agent`, outside any worktree, so that a PDF +converted on one branch is not converted again on the next. `--work` defaults to a new +directory under the system temp directory, which the gate does not delete: the drafts +of a folder that failed are read there. Neither directory is inside the repository, so +`git status` after a gate run reports nothing new. + +`.github/workflows/gate.yml` runs pytest and then the gate on every push to `main` and +every pull request. Mathpix reads the PDF once and the workflow stores the markdown in +the Actions cache under the PDF's hash, so a later run makes no call. The job needs two +repository secrets, `MATHPIX_APP_ID` and `MATHPIX_API_KEY`. A pull request from a fork +is given neither, and its `ci-corpus/pdf` folder faults; the tex and docx folders still +run. + +The `gate` job is a required status check on `main`: + +```sh +gh api -X PUT repos/{owner}/{repo}/branches/main/protection \ + --input .github/branch-protection.json +``` + ## Docker The image carries pandoc, a TeX Live with xelatex able to run the PDF generator's diff --git a/ci-corpus/docx/sheet.docx b/ci-corpus/docx/sheet.docx new file mode 100644 index 0000000000000000000000000000000000000000..137c8d0840d197d2a6afd26ca399d33777145ea1 GIT binary patch literal 11101 zcmZ{KWmsI<5-rfUySuwP!65|K;O_439^BnExO;GS_uxSj+#Mb>cka!E_g;T}y1)Lh z>g=vvwNI^8B_|00MFj$Zgaq2FVAX;&{WcK*3dfaBa?k+h7%r8h&*C;Mb}+ z?7^89VwJdN+Fzx2PE*22+~gd}uhq$(YLafXCs2p0LFw>Nb0W)T*eOOWAd#i4AL4#6 z(*RrovX}}M+;;oCDv8o0Wf^7|6)mSNYJzrg5bYj_j%5mUnn1Xtp@np_sAg==oCkfl zYYNr;_t!@sO{bsfd**u}#V7$4(GS_Z%b`>_hyZQ~pjTBz8JqgcTOG7+E}YC@iOFaJZRg%_A8rx;et5o`bQ zGJf~o{zon**#HO`>-3Lb(gq1Lm@h$ z5?@#AOkG2l^PXur&|sjW5631-dOfyumt_e{4=s+yljdVvsRX(J>kiK;I+cSr+;9gf8lXWYsa2)Q8$Ia z07`dciILB8p|M{=rt7d|m&!(b)gcWV5NDM`^!!1%ty!{G)YI@Gg5-N8lw>dZOv_BB zBAjg!CbBYmjnl9uzdPBAj>_8&GX43WwtM&@BZ8~ch{v$$hI z6W)nB#w~|ucDMFa#l?Y6U!3b8e0c=fz%Yb{=5H%W$l_iS0w#R_0iduG z*po*0NEX}B4*BeYIbpqN!iXi3FZTe4R+9JnaO1V&-*Sck!y%Hyw^yh!jx1%UF;GpH zuVny>!*EL={aQ`)IfxcQlUBs5F1D$gp+nk2A6HZwS4s1W)-9I`)jBJQLoX%ak52Blc&-H)njtcS^r81_XQ#g!61&>A)GxFtpuz(Ix zq3SZ1zq>g%#L`u_xT}I9^cEm+#$-@i0pVWPt)C@!k#!ck(`P|{&ZFa4SnJ!v@5HSH zebHd3F3BO9Y_)Iu8TJq@yNXc}x9msg5jy@v_6RN3eOM~#D!5qGmf2=nW(;Bc4SaCh z`>Ni-_Q+gM79t9G5C3BMHZ|I_N96=h@h*7Qs7Fk7V)k=Sk&wZ-{iA0Kf3>cYu2B(U z$MZ53CcGYKX=A>TiOXfv43-T|rpv6a9mJu@ z{%Zda5L4Mi;I32s%s>Z)XNY%F0>U~8^%cLzJ#>!`Z z^7)JU!&sE5kXiX`Tu;8mJ>`6vWh}#m=kS^*xaOqcu36u!RK4^p&8nKyoasSy_bRSa z5dE1So;mP|T$OL)Ov5LTuPH$-U9QYQMYF!%#2-X@T1a2{kb|r~cH5IT*UtsJlxO*{ zoRAnKDN15n6erC4#&6See@#&r)U!fe{n+ioxljBougaWc?Bc-0A0{bPA*bw97(z+> zqevRu`}%g-6XZ|t1ujRBSiO18^X(|g7?HjdUdjz$iDoqm72PdY*n9)t*1v`Xlk zOBj6wQ}38HyzUhcoH8MNkevR94qC03w%eCB?@MdGRaP`R+YDMvbe$}5|A0X+9!*`X zJ)G0IFS$&g*+`I_5Iw3?M6P35ekxIq5?0MHUo&ZRNxVq=of{rwt-L~DYI^^z$rN6_ zaGT83+}=_tW+_OKQdXRgoq{4HC2yD ziGpm(VGVJnUN>i2ibCjuHGUH2*HJs~4iqN*T4P8kvC6WM1oCF-hE}|-ZhRgLaX3X= zpW&1IR%LNy9QikT63IOKAa_cz>X#G`>ek?4e?;1*euJluVXl=kg9R|CEa%;M8XJ5t z3(Ry^5MtT$`Fyt#<7487tuTzrGQF}JG}vo|V1I!xV~5#e!nyFNJ|MN63q&vv6pDo0 zh5;|bAUcPjfzR=9niVLBnZC3$+`x9PhI+O;P`(}{Pa*gC_Xk~pof>4Y6m2n78=g)@ zjh!+DPJ--mFHbRV&oJ`U(X@Jw~H+oAC+0pKi2Mr#7_ z%yn_@WATnY0ak{}k8}sQm&j#|Av}uxtaKg98#o$k*$OHtbr?7ha6ArA=&LLE=OVn# z*Rk`3+~k|swRi4fE{}JdlaNoYy(j{XhFqNuJ$ab@@KWF2Q&CceV&pFsW4En~j~nf; zt+&xf+Rc|m3*+xf8vzhz03I|D(3Kew5W;_B#lg|d@;{+`tfk?yQjG4+-`O*Iinn3Z zQudQ}utcR!hGMv6?FMU@b3p{18b?PHiCAF?DKba1UO59jrB)^I>F?&$IHj* z6JDVswV{@S=oiczk13|<3HuX<2c5=44)@o|+taLd_nqZn?dV45wx+v0AHqH_9!%Fg zV0^z@2>V=NBagsn-znk_f}H1d<}EblXA;LD-4D3t09Y%5gT%IgUH50xdWzKPEX4H| z%+#q%%KCh~C>PMAS)bQz?P%gvqD$x#KO2#-!DW(f3hX=v{nY#|X#SJL?hzWKvdU}iypXfBf z)tsi8hYW*82|4hixwJq{)V1U2{px5pYrJs*@=sqQa($lloAN zr|bF@iL|i#z3pwtJ67(%<)I19z6cFwatf0L_H9dw#LCCZ_nk+#NR8DJ9Pg;Nphj5G zVL_Chk!&B5js*D{i;`j;_LFWIUVrsmkc_M2JDJ<*!1;6TAB`u%TMN%mBIaW#KlMV4me8S(o@jjt<9;g5zkmfm zdc|mcB|~uM6Y7f66`OI#>w#~V&oB6KnS`m}me$H?gVI3T1aDJEA%j9UD7b^tIQo)6 zK3?1QlG{onuM94Q&M@0%=>WvT-M7mvJ{ElVQxj*GLXS2xD>ZA@e3s?MsA_7@b|cjX z<$~&DT#`#KW|M(T(x>8#vwWSzvv@nEEiN^%9@zvaezWwm_>DkWd$`X%O*LOCCD-?E zhTSe?5{G%L(%WXUliFrs-L4f*WVF*-l^g3X(oCGeq9zi!xGmE2>o#yBRE%@h z*#Xztseb!h96p&$=Mu7*Tv>J5+!30TEcVmvBp99aB-fQ6yVWyz9P%kW{ORu@dj{`x zJBLjWXI5Zl`aDFV37j5%+cF5ZB|x#71Y+#ghOWZvzP2dt*)K>@*BH-$SdjCFuw`;e z3^8qr;+u_RB~UZ#eg^@3tx5o1<2<9iy(*6@GNE&Izmecw!_zTO^ek!Elsry+s09?+ z>{@53*k87RaZKuzr1Sjv)E-B2jgDK=sUYUOX;{2E9xh?6UB9{4Pvdksihr*4eaKVJ zgCpsh(u1pUq&1t2Pl2<7#-p(Du4UEMoNMgV<0kL0XmhRSY+R_hNI5$=dP4W`1uP{gSExi^0Aw8v(tIMe%h$ z->t!{9vNc1v|8DUh3d`C=ISvt|r8#tda#GnIY}b zX41&3>$j)9kxr)T6T5HfiL-96*69}|#49>ZSB+yJoFCumaO0gB14w zqZskjFR+})GEv4;Q&AglriQF3O@ijW9yum6wac2l?#QwC0=*si2AY}RRC48&>_ITv ziGjsY#ep^9l}4rnd4lRk_UvAWD2r47^X2F%UMfD00fwz*d;It-^}z+2J^=BsxIl8X zL@F^)!0CjZ{E;1L(8;w?DGlwn5ec{I4v-I3r@8HS4@Q^?iCPnK0- zSUePoUI}Lz&Be41_rMfGd-6#aGX*34Q8H@!PqNX)#3}OXF9(#z@`|Z(9@txT&=y{f zZlA>)j*qFiW8-vAtyS6!WOJSKisa`eRmr7othf989|=+D#?b9@4-Tb@WVz``72^pW zo&u2a$F0^Qu7aIp6prz~i=c?m# z)rLzD8p{;C)=VmR=U4woGVMgb7O|FLk9$ZAo+_q5yH_2wB{Y1MVfB zyBfks;Xfg$y#k4dv?DUbHAL%1(uI__XXNC2_)zU8Q)dA!D0Y2K%UjfFjd?Z@^cC!g zc$LfUb`wR41&ZiGWQJ?%ydk_hyYV0c8Hr_rlW-#^3OYE~?Av-Z^r9Gso?e7FS!DTx zJB%MV-ZDpLKQJ>ARbCV50hjLnoUIXGtYKz=Zr8nvwO%319nRZA(W4!PJQT=2 zMejY&(>uuC;&pn|0pbtA$ZC7)AGSLV^f@f_ZQ-&TT)edb;sINMmzlF8OJYYo$VPHgbmgK+m#V^Ng)TlQn&_d z9X1Y3C;a#;V(K}z`eAf~s}Ii(gZ`pY%@NQN0B0nwOA*Urn&pdv#7w(g7&~HwYynmzJPEtoRopQCTqz0<_xFKEX-JpBGJOW}|nAj6O zLoftpr)(!bM*p?!i2iVP_nc6G$>HEVauqqH5-g>{a$^Ifu{0;xhoIK+R7+|uj-eKn zm*!Qc0*z!(4+G({DHSL-oj<)uU=2W7eyhXn-lWF=Noxi+R(~mLf0N71>F%&zLPYRQ z6{vN^S>6QEv>SyRs73_B0al@bSS*y=^Tip}gIuy5Qszq^&h7Ij2m>|^{VyG8k6I+u zhj7jl8|rq?&n2Dx_l{B{gud6YWvi!Zh03r@;ACX!-Tr%c;D-(sVjy^ziPp?SPJUCW zdt=>{ooRc7f;(PjKex>a1b%W7kH|i5tp_hW2Kw>?r z?!y&-sl_E@@{t&0OzNn&QC_+&Obpa!Q4`~CSF^D-4v0~-%iDLQvSifme7T?ulgN4j z{?iXz-RBc{Z_y<7k00LT2|0TkTL%UM8+)U_C~RTe_+KJd2Ia9%aZh`Rf91{lr#VvE;swW5YAP5MRihRzNTS4i8 z%EMag1r382fA98=w%h!#BcMAUfl6qoFsUtbD-YbTRC)knhhL6MLYi6vgiYFERoe3LQ$lMG7IHY<#t*zMQx7xJ^YvRE$w~S45cT}Sc1NTYr^v4{? zw(xhS1%7V3#g*&Bqfg6$a!kjLT+8X1Cy+X(JeG*moy30;8_Ic2B%A7S2(Oe|2r`$+ zR;)zeE#-wW7eXn86)lZUD~^C~d%&&0rm71>&M>Hp^kz`eAvI}USP1=I(@vBo-rgt> z*S-l%@nN$$(sx-niG3nqS^uY2bS>vz2?jprp+FC2ca|iazfG5huHd0H8+fs#eaG|} zt;CXZg=ON6z*r?HP6p|){ie|!-g8u-r&kX&ct5PY-JCbYO^FyzZO2$IFAFjojW&TWdi1c)eY~j*-BT(e^lLahOm}-NS})v* zYM>m1`u+eOw&aK7&4S|1)Eoslewv@rpu7{=CVSR=(&USFoQ%KR06BQ;`IwIZ_78Ev zIHnRSYj48tVz(y3 zay0vj)gP#$i!z^A!M4wPY5-^(+-D9c>A2oXWq@#F@ww9cAIO*%HuZ;TCFP7Q23uPiKy7V<3 ziPc<`#0q4h132$g4zI$l>gTukN-!_XN9n1%cDtCqbN{+r;v?$7eIiNHQZBv>>FV|) zHY?#{8^hq<;Ox!yVQey%Vf=_NzI$?vD|#?4m4I_jA>pMq2}j(#Z5^ssrSFr6UX zxm9wubEf0BTgJ0vcHF8UerruKlxz(D7Q~<4D2~Hw{RQ7FwGmS>Phdto zGOueY%NdI?2gg~}Z_ODq7){j=t@UUS)U?KlUq1ntCy#d4n!A1=4HS|%ZnF2*>)tzc zVg4{01SAS^J0;ku_nRd)P!K-ermYgk$?dnGQs8FaEg)p4wI`zZ6`cBn*$*ZrPZWzPQl2k0?G#UtNG}@m$+XPTa!%(g6|Ysjv36!Dj1}9p z2>aF8MYcD0Up!pBjEZ)Ut*lWZ7MRY|F>a1pnJU>>EiLY2TG~qes2gh`Sz7Ht`I>ss z@y>KYvT4RVE8ML0f`l9E1af3++SgSxr6U>q$n?B(DEL8arG&5=BLOJ_57GV9 zg3E3-8mFhme%x;}0)tErs>qtk6;d*#hg+kVc-|gmT z=Vz5JMzm)x$&%j|rS#cJkHZrPF3zXcu>%6~7#!M?xIAkvm{Ds{hl9HFDI#P(Qg@7cXRV zy?z68)Lg0vGkuM7d{Q*(=bO+gYEB^{WWua`gFwf-FPJp@bzgWta^XRrAqJ55xeMRe z>g?pZ*b*YRmnLd`OVFQghsY5Rb?#;7Rv=F0t5Bf-1dv@N*oKj!XyJ<7SIc&j<5;sCp_DEIIztSFwBY)m&AYvXGlxOK z{BYlf2bmQJmx_cK2tFHddM81^^j_m%1CiKGz&h!WuX>t9n2<~1l!cUX8A=2rnl)Ke z3s7;w1z%?^Wke*%t%!zn8E^_fu6CAy&4ldHIFBYT+e4Vqxx?A9 zpVt!`I0VHa!r6CTU^B``e7C;}5rRX)8Q%DMtvmZYx-AOSm$925Y zlk=AQmhp#4QW!Nk@Y7}TS8FqsyUV#Vg<6!@VLlhhQ?ue089L#PT5=pym^3l=AkdnS z%VB`0NXc##Ne?8im%!~UiYN18qng?0D@t_@^E4#2KV+)=7Ge>s;FVc&94Y6{H?9gT zTek+87tT@ETQ*s77?wD+MpA%tN}y|uJGZVl9r>@s9gZ0HQn98LDtA{nr#UbFN?7@- zsriAWzP@L*&*71F%RB$8bWnc&LVjaRze4IFA)2I3UVvvEHr=6UZX(zYpGq|&6n#Ynx*(9+ zFli8t7SnE<5RQ0-2V1%T2H50Mz(ivl6^=-tCPXllxt1BF8(6be#mks^1G}4%;{cFm zl>+WUd;(1yHqjh=$giNhnQJc%HMsS2ZP7$B3;ZC0biQtCDgLjWMXiILZC80cA_?V*C~vsV|fz4UJd_tPeO#}F;A0UQ6#YEJtoa4VD zF+oKaq`*413AfuUp!?8wApHZAqCpQgIRt=MbCk(#>ZP>4M0h9_a@uSM-(=x}^h+gj zN$=Mi1>m>x{Qo~Z_x6|m_44ceK>lOukEwz8`1ifz-}vaaj`H72`Q1r=4}LEf{su3%v-j}#`qFPW1;)P>l7G~u_we_U%5S&>=D*?p6Ik9GcrR`IHb9H@U*G*J zN#DcYv)AA75S)MA{~s><9{rwa{YGQI&9(gP`}e%-y@~ganBOK0N&mNf|9arO$G<JCL-E(I{{z!fiJ|}i literal 0 HcmV?d00001 diff --git a/ci-corpus/pdf/.gitkeep b/ci-corpus/pdf/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ci-corpus/tex/figures/ball.png b/ci-corpus/tex/figures/ball.png new file mode 100644 index 0000000000000000000000000000000000000000..f37764b1f7606623616dcdc169cc858273ea2d94 GIT binary patch literal 70 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx1|;Q0k92}1TpU9xZYBRYe;|OLfu)tPp=D){ QB2a?C)78&qol`;+0Lr!y6951J literal 0 HcmV?d00001 diff --git a/ci-corpus/tex/sheet-1.tex b/ci-corpus/tex/sheet-1.tex new file mode 100644 index 0000000..3127901 --- /dev/null +++ b/ci-corpus/tex/sheet-1.tex @@ -0,0 +1,46 @@ +% Synthetic, in the shape of the corpus's problem sheets: a starred section +% heading, an enumerate of questions with lettered parts, and a solutions +% section at the end. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 1: Kinematics} + +\begin{enumerate} + +\item A ball is thrown straight up at $20\,\mathrm{m/s}$. + \begin{enumerate} + \item Find the greatest height it reaches. + \item Find its time of flight. + \end{enumerate} + +\item A block of mass $m$ rests on a slope of angle $\theta$. + \begin{enumerate} + \item Name the three forces acting on the block. + \item Find the least coefficient of friction that holds it still. + \end{enumerate} + +\end{enumerate} + +\section*{Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item $h = v^2 / 2g = 20.4\,\mathrm{m}$ + \item $t = 2v/g = 4.08\,\mathrm{s}$ + \end{enumerate} + +\item + \begin{enumerate} + \item Weight, the normal reaction, and friction along the slope. + \item $\mu = \tan\theta$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-2.tex b/ci-corpus/tex/sheet-2.tex new file mode 100644 index 0000000..f18652d --- /dev/null +++ b/ci-corpus/tex/sheet-2.tex @@ -0,0 +1,28 @@ +% Synthetic: questions with lettered parts and no solutions on the sheet. The +% solutions are in solutions-2.tex, which is a document of its own, as the +% corpus's problem sheets are. Nothing here is copied from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Fields} + +\begin{enumerate} + +\item A dipole sits at the origin. + \begin{enumerate} + \item Show that $\nabla \cdot \mathbf{B} = 0$ for its field. + \item Find the field on the axis at a distance $z$. + \end{enumerate} + +\item A charge $q$ moves at speed $v$ through a uniform field $\mathbf{B}$. + \begin{enumerate} + \item Find the radius of its circular path. + \item Find the period of the motion. + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/sheet-3.tex b/ci-corpus/tex/sheet-3.tex new file mode 100644 index 0000000..39d12d8 --- /dev/null +++ b/ci-corpus/tex/sheet-3.tex @@ -0,0 +1,50 @@ +% Synthetic: a sheet whose first question includes a figure, which is the shape +% that first failed a sweep on the path of an image. Nothing here is copied +% from ExampleContents. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 3: Statics} + +\begin{enumerate} + +\item The beam below carries a load $W$ at its midpoint. + + \includegraphics[width=0.2\textwidth]{figures/ball.png} + + \begin{enumerate} + \item Find the reaction at each support. + \item Find the bending moment at the midpoint. + \end{enumerate} + +\item A ladder of mass $m$ leans against a smooth wall. + \begin{enumerate} + \item Draw the forces acting on the ladder. + \item Find the least angle at which it does not slip. + \end{enumerate} + +\end{enumerate} + +\section*{Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item $W/2$ at each support. + \item $M = WL/4$ + \end{enumerate} + +\item + \begin{enumerate} + \item The weight at the centre, the normal reaction at the wall, and the + normal reaction and friction at the floor. + \item $\tan\alpha = 1/2\mu$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/ci-corpus/tex/solutions-2.tex b/ci-corpus/tex/solutions-2.tex new file mode 100644 index 0000000..0cd053c --- /dev/null +++ b/ci-corpus/tex/solutions-2.tex @@ -0,0 +1,27 @@ +% Synthetic: the solutions to sheet-2.tex in a file of their own, which is the +% shape that first failed a sweep — a document with no questions in it. +\documentclass[12pt]{article} +\usepackage{amsmath} +\usepackage{graphicx} + +\begin{document} + +\section*{Problem Sheet 2: Solutions} + +\begin{enumerate} + +\item + \begin{enumerate} + \item Take the divergence term by term; each pair cancels. + \item $B = \mu_0 m / 2\pi z^3$ + \end{enumerate} + +\item + \begin{enumerate} + \item $r = mv / qB$ + \item $T = 2\pi m / qB$ + \end{enumerate} + +\end{enumerate} + +\end{document} diff --git a/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml new file mode 100644 index 0000000..54afdc5 --- /dev/null +++ b/corpus-specs/MECH60014_Stress_analysis_3/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Para text~'^(STRESS ANALYSIS|Sheet |Note:)|Description automatically generated' +question: ListItem +part: Para text~'^[A-Z]' +solution: Para text~'^\[' +strip: ['^\d+\.\s+', '^\['] +layout: PartsOneSol diff --git a/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml new file mode 100644 index 0000000..c4bf288 --- /dev/null +++ b/corpus-specs/PHYS40002-Mechanics/problem_sheets_and_figures/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: Para +solution: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/corpus-specs/UCL_MechEng/in2lambda-spec.yaml b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml new file mode 100644 index 0000000..a3c7db3 --- /dev/null +++ b/corpus-specs/UCL_MechEng/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Para text~'Figure [0-9]:' +question: Para label~'^(1a|Q[0-9])' +part: text~'.' +strip: ['^1a\) ', '^Q[0-9]\s*', '^- ', '^[0-9]+\. ', '^i+\)\s*'] +layout: PartsOneSol diff --git a/corpus-specs/baseline.json b/corpus-specs/baseline.json new file mode 100644 index 0000000..1362a08 --- /dev/null +++ b/corpus-specs/baseline.json @@ -0,0 +1,26 @@ +{ + "specs": "corpus-specs", + "folders": { + "ci-corpus/tex": { + "root": ".", + "suffixes": [ + "tex" + ], + "built": 4 + }, + "ci-corpus/docx": { + "root": ".", + "suffixes": [ + "docx" + ], + "built": 1 + }, + "ci-corpus/pdf": { + "root": ".", + "suffixes": [ + "pdf" + ], + "built": 1 + } + } +} diff --git a/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml b/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml new file mode 100644 index 0000000..97c98ce --- /dev/null +++ b/corpus-specs/ci-corpus/docx/in2lambda-spec.yaml @@ -0,0 +1,6 @@ +ignore: Header +question: Para text~'^[A-Z]' +part: ListItem +solution: after Header text=Solutions, Para +strip: ['^\([a-z]\) ', '^\d+\([a-z]\) '] +layout: PartsSepSol diff --git a/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml b/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml new file mode 100644 index 0000000..2a668e6 --- /dev/null +++ b/corpus-specs/ci-corpus/pdf/in2lambda-spec.yaml @@ -0,0 +1,5 @@ +ignore: Header +question: ListItem text~'^[A-Z]' +solution: after Header text=Solutions, ListItem +strip: ['^\d+\. '] +layout: PartsOneSol diff --git a/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml b/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml new file mode 100644 index 0000000..1754146 --- /dev/null +++ b/corpus-specs/ci-corpus/tex/in2lambda-spec.yaml @@ -0,0 +1,4 @@ +ignore: Header +question: ListItem +strip: ['^\d+\.\s+'] +layout: PartsOneSol diff --git a/in2lambda_agent/cli.py b/in2lambda_agent/cli.py index fcde4e6..4f3f9c9 100644 --- a/in2lambda_agent/cli.py +++ b/in2lambda_agent/cli.py @@ -3,10 +3,11 @@ import argparse import getpass import sys +import tempfile from pathlib import Path from typing import Optional, Sequence -from in2lambda_agent import compare, corpus, pipeline +from in2lambda_agent import compare, corpus, gate, pipeline from in2lambda_agent.mathpix import MathpixClient, MathpixError from in2lambda_agent.model import ModelUnavailable, choose_backend from in2lambda_agent.ocr import ocr_pdf @@ -67,7 +68,8 @@ def build_parser() -> argparse.ArgumentParser: """The command line as the design spec describes it. Returns: - A parser with the `run`, `review`, `corpus` and `compare` subcommands. + A parser with the `run`, `review`, `corpus`, `gate` and `compare` + subcommands. """ parser = argparse.ArgumentParser( prog="in2lambda-agent", @@ -199,6 +201,37 @@ def build_parser() -> argparse.ArgumentParser: default=corpus.DEFAULT_SPEC_DIR, help="The tree the sets' specs are kept in, mirroring the corpus.", ) + sweep.add_argument( + "--cache", + type=Path, + default=pipeline.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, so a sweep pointed at a cache " + "another run filled converts nothing.", + ) + + check = subcommands.add_parser( + "gate", help="Replay the corpus the baseline names and check it against it." + ) + check.add_argument("baseline", type=Path, help="The committed baseline file.") + check.add_argument( + "--record", + action="store_true", + help="Write this run's counts to the baseline instead of checking them.", + ) + check.add_argument( + "--cache", + type=Path, + default=gate.DEFAULT_CACHE_DIR, + help="Where the OCR of each PDF is kept, shared between worktrees so " + "that a conversion is paid for once.", + ) + check.add_argument( + "--work", + type=Path, + default=None, + help="Where the folders are copied to be run, under the system temp " + "directory by default so the check writes nothing where it was run.", + ) against = subcommands.add_parser( "compare", help="Check a PDF's OCR against the pages it came from." @@ -241,6 +274,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: specs=args.specs, replay=args.replay, rounds=args.rounds, + cache=args.cache, settings=load_settings(), ) print(f"{len(rows)} documents, written to {args.results}") @@ -249,6 +283,27 @@ def main(argv: Optional[Sequence[str]] = None) -> int: succeeded = {"built", "skipped"} return 0 if rows and all(row.outcome in succeeded for row in rows) else 1 + if args.command == "gate": + baseline = gate.read_baseline(args.baseline) + # The directory is printed and is not deleted, so that the drafts of a + # folder that failed can be read after the run. + work = args.work or Path(tempfile.mkdtemp(prefix="in2lambda-agent-gate-")) + print(f"work {work}") + report = gate.run( + baseline, + record=args.record, + cache=args.cache, + work=work, + settings=load_settings(), + ) + for name, summary in report.folders.items(): + print(gate.folder_line(name, summary)) + if args.record: + gate.write_baseline(baseline, args.baseline) + print(f"recorded {args.baseline}") + return 0 + return 1 if report.failed else 0 + if args.command == "compare": settings = load_settings() try: diff --git a/in2lambda_agent/corpus.py b/in2lambda_agent/corpus.py index 95fb16e..7ff1f46 100644 --- a/in2lambda_agent/corpus.py +++ b/in2lambda_agent/corpus.py @@ -249,6 +249,7 @@ def run_one( settings: Settings, rounds: int = 3, replay: bool = False, + cache: Path = pipeline.DEFAULT_CACHE_DIR, backend: Optional[Backend] = None, ) -> Row: """Runs the pipeline over one document and reads the row off what it did. @@ -265,6 +266,7 @@ def run_one( settings: The environment the run has available. rounds: The round limit, ignored in a replay, which can run none. replay: Run the saved spec and nothing else, making no model call. + cache: Where the OCR of each PDF is kept. backend: The backend to write a spec with, chosen from the settings if absent. @@ -285,6 +287,7 @@ def run_one( # report: the row then says what the saved spec left rather than # that a call could not be made. rounds=0 if replay else rounds, + cache_dir=cache, backend=NoModel() if replay else backend, ) except ModelUnavailable as error: @@ -362,6 +365,7 @@ def sweep( specs: Path = DEFAULT_SPEC_DIR, replay: bool = False, rounds: int = 3, + cache: Path = pipeline.DEFAULT_CACHE_DIR, settings: Optional[Settings] = None, backend: Optional[Backend] = None, ) -> list[Row]: @@ -376,6 +380,8 @@ def sweep( specs: The tree the sets' specs are kept in, mirroring the corpus. replay: Run the saved specs and nothing else, making no model call. rounds: The round limit each run is given. + cache: Where the OCR of each PDF is kept, so that a sweep pointed at a + cache another run filled converts nothing. settings: The environment the runs have available. backend: The backend to write the specs with, chosen from the settings if absent. @@ -455,6 +461,7 @@ def sweep( settings=settings, rounds=rounds, replay=replay, + cache=cache, backend=backend, ) print(f"{row.outcome:<20} {row.source}") diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py new file mode 100644 index 0000000..97c0a0e --- /dev/null +++ b/in2lambda_agent/gate.py @@ -0,0 +1,219 @@ +"""The merge gate: a replay over real documents, compared with a baseline. + +Every ticket before this one was tested on synthetic fixtures. The first sweep +over a folder of real PDFs failed on four faults no fixture had: a list-valued +selector, a file of solutions with no questions, an image path, and a wrapped +line. A merge now requires an end-to-end run over real documents. + +The run is `corpus.sweep(replay=True)` over the specs the repository keeps, so +it makes no model call. The baseline file names the folders to run and records, +per folder, how many documents built. The gate fails when a folder builds fewer +documents than the baseline records. A folder the baseline records no count for +passes on any count, which is how a folder is added to the gate before it +replays to a build worth defending. The baseline is committed, and a change to +it belongs in a pull request that states why the counts changed. +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from in2lambda_agent import corpus +from in2lambda_agent.settings import Settings + +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "in2lambda-agent" +"""Where the gate reads the OCR of each PDF. The directory is outside every +worktree, so a PDF converted on one branch is not converted again on the next. +`corpus` on its own keeps its cache under the directory the user ran from.""" + + +@dataclass +class Folder: + """One folder of a corpus, as the baseline holds it. + + Attributes: + root: The corpus directory the folder is under. The repository itself + for `ci-corpus`, which is committed; an absolute path for a corpus + outside the repository. + suffixes: The file suffixes that are documents in the folder: `pdf` for + a folder of scans, `tex` or `docx` for sources. + built: How many documents built when the baseline was recorded, or None + where no count is recorded. `root` and `suffixes` are written by + hand; `built` is what `--record` writes. + """ + + root: Path + suffixes: list[str] + built: Optional[int] = None + + +@dataclass +class Baseline: + """The committed file the gate compares a sweep against. + + Attributes: + specs: The tree the folders' specs are kept in, relative to the + repository. Folder `A/B` reads `/A/B/in2lambda-spec.yaml`. + folders: The folders to run, by their path under their own root. + """ + + specs: Path + folders: dict[str, Folder] + + +@dataclass +class Summary: + """What one folder did this run, beside what the baseline records. + + Attributes: + built: How many documents built. + counts: How many did each other thing, by outcome. + recorded: How many built when the baseline was recorded, or None where + no count is recorded. + """ + + built: int = 0 + counts: dict[str, int] = field(default_factory=dict) + recorded: Optional[int] = None + + @property + def failed(self) -> bool: + """Whether this folder fails the gate.""" + return self.recorded is not None and self.built < self.recorded + + +@dataclass +class Report: + """The whole run: one summary per folder, in the baseline's order.""" + + folders: dict[str, Summary] = field(default_factory=dict) + + @property + def failed(self) -> bool: + """Whether any folder fails the gate.""" + return any(one.failed for one in self.folders.values()) + + +def read_baseline(path: Path) -> Baseline: + """Reads the committed baseline. + + Args: + path: The JSON file. + + Returns: + The baseline. + """ + written = json.loads(Path(path).read_text(encoding="utf-8")) + return Baseline( + specs=Path(written["specs"]), + folders={ + name: Folder( + root=Path(one["root"]), + suffixes=list(one["suffixes"]), + built=one.get("built"), + ) + for name, one in written["folders"].items() + }, + ) + + +def write_baseline(baseline: Baseline, path: Path) -> None: + """Writes the baseline back, counts and all. + + Args: + baseline: What to write. + path: The JSON file, which is overwritten. + """ + written = { + "specs": baseline.specs.as_posix(), + "folders": { + name: { + "root": one.root.as_posix(), + "suffixes": one.suffixes, + "built": one.built, + } + for name, one in baseline.folders.items() + }, + } + Path(path).write_text(json.dumps(written, indent=2) + "\n", encoding="utf-8") + + +def run( + baseline: Baseline, + *, + record: bool = False, + cache: Path, + work: Path, + settings: Optional[Settings] = None, +) -> Report: + """Replays every folder the baseline names and counts what each one built. + + Nothing is written into the repository or into a corpus: each folder is + swept into its own directory under `work`, and the table is written there. + + Args: + baseline: The folders to run and the counts to compare against. In + record mode this run's counts replace them. + record: Take this run as the new baseline rather than checking it. + cache: Where the OCR of each PDF is kept. The directory is shared + between worktrees, so Mathpix converts each PDF once. + work: Where the folders are copied to be run. + settings: The environment the runs have available. + + Returns: + One summary per folder. + """ + work = Path(work) + report = Report() + for name, folder in baseline.folders.items(): + rows = corpus.sweep( + folder.root, + paths=[Path(name)], + suffixes=folder.suffixes, + results=work / name / "results.csv", + work=work / "work", + specs=baseline.specs, + replay=True, + cache=cache, + settings=settings, + ) + summary = Summary(recorded=folder.built) + for row in rows: + if row.outcome == "built": + summary.built += 1 + else: + summary.counts[row.outcome] = summary.counts.get(row.outcome, 0) + 1 + if record: + folder.built = summary.built + # A recording run reports what this sweep did, so it fails nothing. + summary.recorded = summary.built + report.folders[name] = summary + return report + + +def folder_line(name: str, summary: Summary) -> str: + """The gate's one line for a folder: what it did, beside what is recorded. + + Args: + name: The folder, as the baseline names it. + summary: What it did. + + Returns: + The line. + """ + counts = " ".join( + f"{outcome} {summary.counts.get(outcome, 0)}" + for outcome in ("faulted", "build refused", "skipped") + ) + other = sorted(set(summary.counts) - {"faulted", "build refused", "skipped"}) + recorded = ( + "not recorded" + if summary.recorded is None + else f"baseline built {summary.recorded}" + ) + return ( + f"{name:<40} built {summary.built} {counts}" + + "".join(f" {outcome} {summary.counts[outcome]}" for outcome in other) + + f" ({recorded})" + ) diff --git a/in2lambda_agent/ocr.py b/in2lambda_agent/ocr.py index b34f088..707ad2c 100644 --- a/in2lambda_agent/ocr.py +++ b/in2lambda_agent/ocr.py @@ -9,6 +9,7 @@ import shutil from dataclasses import dataclass from pathlib import Path +from typing import Optional from in2lambda_agent.mathpix import MathpixClient @@ -25,6 +26,27 @@ class OcrResult: fresh: bool +def cached(pdf: Path, cache_dir: Path) -> Optional[OcrResult]: + """The conversion already in the cache, or None where there is none. + + Asked before a client is built, so that a document whose OCR was fetched + once runs again with no Mathpix credentials at all: a worktree, or a CI job + on a fork, has the cache and not the account. + + Args: + pdf: The PDF whose conversion is wanted. + cache_dir: Holds one entry per document, named by the PDF's hash. + + Returns: + Where the markdown and its media folder are, or None. + """ + entry = Path(cache_dir) / _hash(pdf) + markdown = entry / SOURCE_NAME + if not markdown.exists(): + return None + return OcrResult(markdown, entry / MEDIA_NAME, fresh=False) + + def ocr_pdf( pdf: Path, *, cache_dir: Path, client: MathpixClient, fresh: bool = False ) -> OcrResult: @@ -42,11 +64,12 @@ def ocr_pdf( Raises: MathpixError: If the conversion fails; the entry is left absent. """ + if not fresh and (hit := cached(pdf, cache_dir)) is not None: + return hit + entry = Path(cache_dir) / _hash(pdf) markdown = entry / SOURCE_NAME media = entry / MEDIA_NAME - if markdown.exists() and not fresh: - return OcrResult(markdown, media, fresh=False) # A fresh pass restarts the pipeline for this document, so the whole entry # goes: anything a later stage comes to keep beside source.md — a draft, a diff --git a/in2lambda_agent/pipeline.py b/in2lambda_agent/pipeline.py index 7045a03..55dbcfc 100644 --- a/in2lambda_agent/pipeline.py +++ b/in2lambda_agent/pipeline.py @@ -34,7 +34,7 @@ from in2lambda_agent.fix import RoundResult, fix_round, summary from in2lambda_agent.mathpix import MathpixClient from in2lambda_agent.model import Backend, ModelUnavailable, Usage, choose_backend -from in2lambda_agent.ocr import ocr_pdf +from in2lambda_agent.ocr import cached, ocr_pdf from in2lambda_agent.review import RECORD, Question, Review, choose from in2lambda_agent.settings import Settings from in2lambda_agent.spec import RECORD_NAME, record_run, spec_path, write_spec @@ -125,7 +125,8 @@ def run( Raises: MathpixError: If a PDF cannot be converted, MissingCredentials among - them when the run has no Mathpix credentials. + them when a conversion is needed and the run has no Mathpix + credentials. A PDF already in the cache needs none. ModelUnavailable: If a spec must be written and no backend can run. BadSpec: If what the model answers with is not a spec. SpecRejected: If in2lambda will not run the spec. @@ -145,8 +146,13 @@ def run( # The rest of the pipeline reads markdown, so a PDF becomes markdown first. if source.suffix.lower() == ".pdf": - client = mathpix or MathpixClient.from_settings(settings) - ocr = ocr_pdf(source, cache_dir=cache_dir, client=client, fresh=fresh_ocr) + # The cache is asked before the client is built: a document converted + # once runs again with no credentials, which is what lets a worktree or + # a fork's CI job replay a corpus of PDFs it cannot pay for. + ocr = None if fresh_ocr else cached(source, cache_dir) + if ocr is None: + client = mathpix or MathpixClient.from_settings(settings) + ocr = ocr_pdf(source, cache_dir=cache_dir, client=client, fresh=fresh_ocr) frozen = ocr.markdown # A fresh pass is a restart: every stage below reads the new markdown. message = ( diff --git a/tests/test_cli.py b/tests/test_cli.py index 740d234..5020c2d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,7 +7,7 @@ import pytest from conftest import FakeMathpix -from in2lambda_agent import cli, compare, corpus, pipeline +from in2lambda_agent import cli, compare, corpus, gate, pipeline from in2lambda_agent.cli import build_parser, main, reviewer_name from in2lambda_agent.model import Usage from in2lambda_agent.settings import Settings @@ -90,6 +90,7 @@ def test_corpus_defaults(): assert args.results == Path("results.csv") assert args.work == Path(".in2lambda-agent/corpus") assert args.specs == Path("corpus-specs") + assert args.cache == Path(".in2lambda-agent") def test_corpus_every_option(): @@ -112,6 +113,8 @@ def test_corpus_every_option(): "working", "--specs", "saved", + "--cache", + "cached", ] ) @@ -122,6 +125,104 @@ def test_corpus_every_option(): assert args.results == Path("sweep.csv") assert args.work == Path("working") assert args.specs == Path("saved") + assert args.cache == Path("cached") + + +def test_the_corpus_cache_is_handed_to_the_sweep(monkeypatch): + given = {} + + def record(*args, **kwargs): + given.update(kwargs) + return [] + + monkeypatch.setattr(corpus, "sweep", record) + + main(["corpus", "ExampleContents", "--cache", "cached"]) + + assert given["cache"] == Path("cached") + + +def test_gate_defaults(): + args = build_parser().parse_args(["gate", "corpus-specs/baseline.json"]) + + assert args.command == "gate" + assert args.baseline == Path("corpus-specs/baseline.json") + assert args.record is False + assert args.cache == Path.home() / ".cache" / "in2lambda-agent" + # Chosen when the command runs, so that two runs do not share a directory. + assert args.work is None + + +def test_gate_every_option(): + args = build_parser().parse_args( + [ + "gate", + "saved.json", + "--record", + "--cache", + "cached", + "--work", + "working", + ] + ) + + assert args.record is True + assert args.cache == Path("cached") + assert args.work == Path("working") + + +def test_a_gate_that_passes_exits_zero(tmp_path, monkeypatch, capsys): + path = written_baseline(tmp_path) + report = gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + monkeypatch.setattr(gate, "run", lambda *args, **kwargs: report) + + code = main(["gate", str(path)]) + + assert code == 0 + assert "tex" in capsys.readouterr().out + + +def test_a_gate_that_fails_exits_one_and_says_what_the_folder_built( + tmp_path, monkeypatch, capsys +): + path = written_baseline(tmp_path) + summary = gate.Summary(built=1, counts={"faulted": 1}, recorded=2) + monkeypatch.setattr( + gate, "run", lambda *args, **kwargs: gate.Report(folders={"tex": summary}) + ) + + code = main(["gate", str(path)]) + + assert code == 1 + assert "tex" in capsys.readouterr().out + + +def test_recording_writes_the_baseline_and_exits_zero(tmp_path, monkeypatch): + path = written_baseline(tmp_path) + + def record(baseline, **kwargs): + baseline.folders["tex"].built = 2 + return gate.Report(folders={"tex": gate.Summary(built=2, recorded=2)}) + + monkeypatch.setattr(gate, "run", record) + + code = main(["gate", str(path), "--record"]) + + assert code == 0 + assert gate.read_baseline(path).folders["tex"].built == 2 + + +def written_baseline(tmp_path): + """A baseline file on disk, for the gate command to read.""" + path = tmp_path / "baseline.json" + gate.write_baseline( + gate.Baseline( + specs=Path("corpus-specs"), + folders={"tex": gate.Folder(root=tmp_path / "corpus", suffixes=["tex"])}, + ), + path, + ) + return path def test_compare_defaults(): diff --git a/tests/test_corpus.py b/tests/test_corpus.py index 70a1376..99bf18f 100644 --- a/tests/test_corpus.py +++ b/tests/test_corpus.py @@ -98,6 +98,22 @@ def test_the_row_says_what_the_spec_made_of_the_document(root, tmp_path): assert sheet.model_seconds == 0.0 and sheet.wall_seconds > 0 +def test_the_sweeps_cache_is_where_each_run_looks_for_its_ocr( + root, tmp_path, monkeypatch +): + given = [] + + def record(source, **kwargs): + given.append(kwargs["cache_dir"]) + raise RuntimeError("as far as this goes") + + monkeypatch.setattr(pipeline, "run", record) + + rows = sweep(root, tmp_path, cache=tmp_path / "shared") + + assert given == [tmp_path / "shared"] * len(rows) + + def test_a_saved_spec_and_its_source_replay_with_no_model_call(root, tmp_path): for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): saved = tmp_path / "specs" / folder / SPEC_NAME diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..d27aad2 --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,198 @@ +"""The merge gate: a replay over a corpus, checked against a recorded baseline.""" + +from pathlib import Path + +import pytest +from test_corpus import make_set +from test_pipeline import SPEC, TEX_SPEC + +from in2lambda_agent import gate +from in2lambda_agent.gate import Baseline, Folder +from in2lambda_agent.settings import Settings +from in2lambda_agent.spec import SPEC_NAME + + +@pytest.fixture +def corpus_root(tmp_path): + """A corpus of two folders, with a saved spec each so a replay builds.""" + made = tmp_path / "corpus" + make_set(made, "sheets", ["sheet.md", "sheet-2.md"]) + make_set(made, "tex", ["tex-sheet.tex", "tex-sheet-2.tex"]) + for folder, text in (("sheets", SPEC), ("tex", TEX_SPEC)): + saved = tmp_path / "specs" / folder / SPEC_NAME + saved.parent.mkdir(parents=True) + saved.write_text(text) + return made + + +@pytest.fixture +def baseline(corpus_root, tmp_path): + """The baseline as it is written by hand, before anything is recorded.""" + return Baseline( + specs=tmp_path / "specs", + folders={ + "sheets": Folder(root=corpus_root, suffixes=["md"]), + "tex": Folder(root=corpus_root, suffixes=["tex"]), + }, + ) + + +def run(baseline, tmp_path, **kwargs): + """A gate run with its cache and work directory under tmp_path.""" + return gate.run( + baseline, + cache=tmp_path / "cache", + work=tmp_path / "gate", + settings=Settings(), + **kwargs, + ) + + +def test_recording_fills_the_counts_and_leaves_what_was_written_by_hand( + baseline, corpus_root, tmp_path +): + report = run(baseline, tmp_path, record=True) + + assert not report.failed + assert baseline.folders["sheets"].built == 2 + assert baseline.folders["tex"].built == 2 + # The hand-written half is the run's to read, not to write over. + assert baseline.folders["tex"].root == corpus_root + assert baseline.folders["tex"].suffixes == ["tex"] + + +def test_a_recorded_baseline_passes_the_run_that_recorded_it(baseline, tmp_path): + run(baseline, tmp_path, record=True) + + report = run(baseline, tmp_path) + + assert not report.failed + assert [one.built for one in report.folders.values()] == [2, 2] + + +def test_a_folder_that_builds_fewer_than_recorded_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built += 1 + + report = run(baseline, tmp_path) + + assert report.failed + assert report.folders["tex"].failed and not report.folders["sheets"].failed + + +def test_a_folder_whose_documents_no_longer_build_fails(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # The spec a replay has nothing to replay without. + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert report.failed + summary = report.folders["tex"] + assert (summary.built, summary.recorded) == (0, 2) + assert summary.counts == {"no spec": 2} + + +def test_a_folder_the_baseline_records_no_count_for_passes(baseline, tmp_path): + run(baseline, tmp_path, record=True) + # A folder added to the gate before it replays to a build worth defending. + baseline.folders["tex"].built = None + (tmp_path / "specs" / "tex" / SPEC_NAME).unlink() + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["tex"].built == 0 + + +def test_a_folder_that_builds_more_than_recorded_passes(baseline, corpus_root, tmp_path): + run(baseline, tmp_path, record=True) + (corpus_root / "sheets" / "sheet-3.md").write_text( + (corpus_root / "sheets" / "sheet.md").read_text() + ) + + report = run(baseline, tmp_path) + + assert not report.failed + assert report.folders["sheets"].built == 3 + + +def test_the_folder_line_says_what_each_outcome_came_to(baseline, tmp_path): + run(baseline, tmp_path, record=True) + summary = run(baseline, tmp_path).folders["sheets"] + + line = gate.folder_line("sheets", summary) + + assert line.split() == [ + "sheets", + "built", + "2", + "faulted", + "0", + "build", + "refused", + "0", + "skipped", + "0", + "(baseline", + "built", + "2)", + ] + + +def test_the_folder_line_names_an_outcome_of_its_own(baseline, tmp_path): + summary = gate.Summary(built=1, counts={"no spec": 2}, recorded=3) + + line = gate.folder_line("tex", summary) + + assert "no spec 2" in line and "(baseline built 3)" in line + + +def test_the_folder_line_says_where_no_count_is_recorded(baseline, tmp_path): + line = gate.folder_line("tex", gate.Summary(built=1)) + + assert line.endswith("(not recorded)") + + +def test_the_baseline_survives_being_written_and_read(baseline, tmp_path): + run(baseline, tmp_path, record=True) + baseline.folders["tex"].built = None + path = tmp_path / "baseline.json" + + gate.write_baseline(baseline, path) + read = gate.read_baseline(path) + + assert read == baseline + + +def test_a_run_writes_nothing_under_the_directory_it_was_run_from( + baseline, tmp_path, monkeypatch +): + ran_from = tmp_path / "empty" + ran_from.mkdir() + monkeypatch.chdir(ran_from) + + run(baseline, tmp_path) + + assert list(ran_from.iterdir()) == [] + + +def test_the_gates_cache_is_not_the_one_a_worktree_would_fill(tmp_path): + # Shared between worktrees on purpose: OCR already fetched for a PDF is not + # fetched again on the next branch. + assert gate.DEFAULT_CACHE_DIR == Path.home() / ".cache" / "in2lambda-agent" + + +def test_the_committed_baseline_names_the_ci_corpus_and_its_specs(): + repository = Path(__file__).resolve().parent.parent + baseline = gate.read_baseline(repository / "corpus-specs" / "baseline.json") + + assert set(baseline.folders) == { + "ci-corpus/tex", + "ci-corpus/docx", + "ci-corpus/pdf", + } + for name, folder in baseline.folders.items(): + # The spec each folder replays, at the path the sweep reads it from. + assert (repository / baseline.specs / name / SPEC_NAME).is_file() + assert folder.built is not None diff --git a/tests/test_ocr.py b/tests/test_ocr.py index cbb1355..97caafc 100644 --- a/tests/test_ocr.py +++ b/tests/test_ocr.py @@ -8,7 +8,7 @@ import pytest from in2lambda_agent.mathpix import MathpixError -from in2lambda_agent.ocr import ocr_pdf +from in2lambda_agent.ocr import cached, ocr_pdf from conftest import FakeMathpix @@ -95,6 +95,19 @@ def test_the_markdown_is_written_as_utf8_under_any_locale(pdf, tmp_path): assert written.decode("utf-8") == "# Ångström ½\n" +def test_a_document_not_in_the_cache_is_no_hit(pdf, tmp_path): + assert cached(pdf, tmp_path / "cache") is None + + +def test_a_cached_document_is_a_hit_without_a_client(pdf, tmp_path): + written = ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix()) + + hit = cached(pdf, tmp_path / "cache") + + assert hit is not None and not hit.fresh + assert (hit.markdown, hit.media) == (written.markdown, written.media) + + def test_each_document_gets_its_own_entry(pdf, tmp_path): other = tmp_path / "other.pdf" other.write_bytes(b"%PDF-1.4 a different sheet") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 1b50a15..5639b29 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -14,6 +14,7 @@ from in2lambda_agent import package, pipeline from in2lambda_agent.cli import main from in2lambda_agent.model import ModelUnavailable +from in2lambda_agent.ocr import ocr_pdf from in2lambda_agent.package import SpecRejected from in2lambda_agent.review import ReviewError from in2lambda_agent.settings import Settings @@ -1255,6 +1256,26 @@ def test_a_second_run_over_the_same_pdf_uses_the_cache(pdf, tmp_path): assert len(backend.calls) == 1 +def test_a_cached_pdf_runs_with_no_mathpix_credentials(pdf, tmp_path, monkeypatch): + monkeypatch.delenv("MATHPIX_APP_ID", raising=False) + monkeypatch.delenv("MATHPIX_API_KEY", raising=False) + ocr_pdf(pdf, cache_dir=tmp_path / "cache", client=FakeMathpix(SOURCE.read_text())) + + # No client at all: the cache is what the run reads, and building one from + # empty settings would raise before it got there. + result = pipeline.run( + pdf, + out_dir=tmp_path / "out", + settings=Settings(), + cache_dir=tmp_path / "cache", + backend=FakeBackend(SPEC), + ) + ocr = next(stage for stage in result.stages if stage.name == "ocr") + + assert ocr.message.startswith("cached ") + assert result.zip_path.exists() + + def test_a_pdf_without_credentials_exits_one_naming_the_variables( pdf, tmp_path, monkeypatch, capsys ): From f54e469badb2af76e0b12e80cde0b28350f356c4 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Mon, 21 Sep 2026 01:34:01 +0100 Subject: [PATCH 2/2] implement: Gate every merge on a replay sweep over the corpus (t21) --- in2lambda_agent/gate.py | 15 +++++++++------ tests/test_gate.py | 4 +++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/in2lambda_agent/gate.py b/in2lambda_agent/gate.py index 8d5ebc4..35ef428 100644 --- a/in2lambda_agent/gate.py +++ b/in2lambda_agent/gate.py @@ -32,10 +32,11 @@ worktree, so a PDF converted on one branch is not converted again on the next. `corpus` on its own keeps its cache under the directory the user ran from.""" -RANK = {"built": 0, "skipped": 0, "build refused": 1, "faulted": 2} -"""How bad each outcome is. A document that did not run at all — a spec gone, a -file unreadable, a document the sweep no longer finds — is worse than any of -these, and ranks below the table.""" +RANK = {"built": 0, "build refused": 1, "faulted": 2, "skipped": 3} +"""How bad each outcome is. `skipped` is the worst of them: a document that +faults was at least read, and one that is skipped was not. Anything else is a +document that did not run either — a spec gone, a file unreadable, a document +the sweep no longer finds — and ranks with `skipped`.""" MISSING = "missing" """What a recorded document the sweep no longer finds is compared as.""" @@ -44,8 +45,10 @@ def worse(current: str, recorded: str) -> bool: """Whether a document did worse this run than the baseline records. - `built` and `skipped` are the same rank: a document in no suffix the folder - runs is not a document that stopped building. + A document that stops being read is a regression whatever it did before, + which is the case a baseline of nothing but faults rests on: widen what + counts as a solutions file and a document leaves the sweep as `skipped` + without a single count changing. Args: current: What the document did this run. diff --git a/tests/test_gate.py b/tests/test_gate.py index a3d4ab1..6dd7921 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -167,7 +167,9 @@ def test_a_baseline_of_no_builds_still_notices_a_document_that_did_worse( [ ("built", "built", False), ("built", "faulted", False), - ("skipped", "built", False), + ("skipped", "built", True), + # A document read and faulted, then not read at all. + ("skipped", "faulted", True), ("build refused", "built", True), ("faulted", "built", True), ("faulted", "build refused", True),