From 52e598aa609934883af0fc314d4febff8a6df0ad Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:28:51 +0700 Subject: [PATCH] Measure what actually works under Pyodide, and answer gate Q1 Every Tier 0 experiment in this project is supposed to run in a browser tab with nothing installed, which rests on Pyodide keeping the introspection surfaces the lessons poke at. Nobody had checked. This checks, on every pull request. `wasmprobe` runs fifteen questions twice, once on a native CPython and once inside a real WebAssembly runtime driven from Node, and puts the two answers side by side. Twelve behave identically. `_testinternalcapi` imports, `compiler_codegen` and `optimize_cfg` both run, `ctypes` reads the reference count and the type pointer off a live object, `sys.monitoring` fires, `sys.settrace` still reports call, line and return, the cycle collector frees a cycle, and the whole front end imports and disassembles to the same bytes. Three do not, and none of them takes an experiment out of Tier 0. The metadata `compiler_codegen` returns in that build has no `consts` key, so the one line in `pyxray.compiler.stages` that asks for it raises, while the optimizer itself is fine once you build the list yourself. Handing `optimize_cfg` a list that is too short reads past the end of memory and kills the runtime, where a native interpreter raises a clean ValueError. And a thread cannot be started, which the concurrency lessons already assumed. The driver runs the checks one at a time and boots a fresh runtime after one takes the old one down, because otherwise every check after the crash looks broken. A Tier 0 check can carry an `accepted` sentence saying we know it fails and what we do instead, so a known gap stays in the report without leaving the build permanently red and hiding the next one. `probes/pyodide` has the matrix, the two raw runs, the written decision, and a notebook that asks your own browser the same questions rather than asking you to trust a recording of ours. --- .github/workflows/ci.yml | 43 +++ .gitignore | 3 + CONTRIBUTING.md | 8 + README.md | 3 + justfile | 23 +- probes/pyodide/decision.md | 61 +++ probes/pyodide/native.json | 160 ++++++++ probes/pyodide/probe.ipynb | 355 ++++++++++++++++++ probes/pyodide/pyodide.json | 152 ++++++++ probes/pyodide/report.md | 70 ++++ pyproject.toml | 7 + tools/wasmprobe/README.md | 37 ++ tools/wasmprobe/driver.mjs | 117 ++++++ tools/wasmprobe/package-lock.json | 55 +++ tools/wasmprobe/package.json | 13 + tools/wasmprobe/pyproject.toml | 17 + tools/wasmprobe/src/wasmprobe/__init__.py | 36 ++ tools/wasmprobe/src/wasmprobe/browser.py | 87 +++++ tools/wasmprobe/src/wasmprobe/checks.py | 350 +++++++++++++++++ tools/wasmprobe/src/wasmprobe/cli.py | 159 ++++++++ tools/wasmprobe/src/wasmprobe/native.py | 40 ++ tools/wasmprobe/src/wasmprobe/notebook.py | 172 +++++++++ tools/wasmprobe/src/wasmprobe/report.py | 157 ++++++++ tools/wasmprobe/src/wasmprobe/result.py | 101 +++++ .../wasmprobe/tests/test_wasmprobe_browser.py | 139 +++++++ .../wasmprobe/tests/test_wasmprobe_checks.py | 82 ++++ tools/wasmprobe/tests/test_wasmprobe_cli.py | 111 ++++++ .../wasmprobe/tests/test_wasmprobe_native.py | 66 ++++ .../tests/test_wasmprobe_notebook.py | 102 +++++ .../wasmprobe/tests/test_wasmprobe_report.py | 153 ++++++++ .../wasmprobe/tests/test_wasmprobe_result.py | 52 +++ uv.lock | 8 + 32 files changed, 2938 insertions(+), 1 deletion(-) create mode 100644 probes/pyodide/decision.md create mode 100644 probes/pyodide/native.json create mode 100644 probes/pyodide/probe.ipynb create mode 100644 probes/pyodide/pyodide.json create mode 100644 probes/pyodide/report.md create mode 100644 tools/wasmprobe/README.md create mode 100644 tools/wasmprobe/driver.mjs create mode 100644 tools/wasmprobe/package-lock.json create mode 100644 tools/wasmprobe/package.json create mode 100644 tools/wasmprobe/pyproject.toml create mode 100644 tools/wasmprobe/src/wasmprobe/__init__.py create mode 100644 tools/wasmprobe/src/wasmprobe/browser.py create mode 100644 tools/wasmprobe/src/wasmprobe/checks.py create mode 100644 tools/wasmprobe/src/wasmprobe/cli.py create mode 100644 tools/wasmprobe/src/wasmprobe/native.py create mode 100644 tools/wasmprobe/src/wasmprobe/notebook.py create mode 100644 tools/wasmprobe/src/wasmprobe/report.py create mode 100644 tools/wasmprobe/src/wasmprobe/result.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_browser.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_checks.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_cli.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_native.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_notebook.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_report.py create mode 100644 tools/wasmprobe/tests/test_wasmprobe_result.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f760b64..255cb20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,49 @@ jobs: path: build/versions - run: uv run nbversion compare build/versions/3.15 build/versions/3.14 + # Everything in the browser tier rests on Pyodide keeping the surfaces the lessons poke + # at, and nobody notices when one of them goes away between releases. This job asks, on + # every pull request, rather than the once when somebody thought to check. + probe: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - uses: actions/setup-node@v5 + with: + node-version: "22" + cache: npm + cache-dependency-path: tools/wasmprobe/package-lock.json + - run: uv sync --all-packages + # The committed pair first, which is cheap and catches the common failure: somebody + # edited a check and did not rerun `just build-probe`, so the report and the notebook + # next to it now describe something else. + - run: uv run wasmprobe check probes/pyodide + - run: uv run nbcheck run probes + # Then the real thing. `npm ci` installs the pinned Pyodide, and the two runs below + # answer the question again against whatever the world looks like today rather than + # against a recording from whenever somebody last looked. + - run: npm ci + working-directory: tools/wasmprobe + # 3.14 rather than the pin, because Pyodide ships 3.14. A native 3.15 control would + # report every version difference as a WebAssembly one. + - run: uv run --python 3.14 --all-packages wasmprobe native --into build/probe + env: + # Its own environment, or this overwrites the .venv the steps above are using and + # the browser run below finds a 3.14 interpreter it did not ask for. + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/venv-314 + - run: uv run wasmprobe browser --into build/probe + - run: uv run wasmprobe report build/probe --into build/probe/report.md + - run: uv run wasmprobe notebook --into build/probe/probe.ipynb + - run: uv run wasmprobe check build/probe + - uses: actions/upload-artifact@v4 + with: + name: probe + path: build/probe + retention-days: 7 + blueprints: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index a4e959c..cc2c5fc 100644 --- a/.gitignore +++ b/.gitignore @@ -218,4 +218,7 @@ __marimo__/ .streamlit/secrets.toml vendor/ .venv/ + +# Node, which only the WebAssembly probe driver needs +node_modules/ citations.lock.json.bak diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df60e7d..172911e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,6 +59,14 @@ There is a second keyword for the other kind of cell. `varies=` is for output th When the differing cell is the lesson's central observation, the note is not enough. Either the lesson gets a short section explaining both versions, because the difference is itself worth teaching, or the example changes to one that behaves the same on both. Which of the two depends on whether the difference is interesting. `LOAD_COMMON_CONSTANT` is interesting and gets explained. A line number inside `asyncio` is not, and the cell should stop printing it. +## What the browser can and cannot do + +A Tier 0 experiment has to run in a browser tab with nothing installed. Which surfaces survive that is measured, not assumed, and the answer lives in [probes/pyodide](probes/pyodide): a matrix, the two raw runs behind it, a notebook you can open in Colab to ask your own runtime the same questions, and a written decision. + +Read `decision.md` before writing an experiment that pokes at the interpreter. Three things are known to be different today. `optimize_cfg` cannot be handed the constants list `compiler_codegen` returns, because that build does not put one there. Handing `optimize_cfg` a constants list that is too short reads past the end of memory and kills the runtime, where a native interpreter raises a tidy `ValueError`, so anything that builds one has to build it correctly rather than catch the mistake. And a thread cannot be started. + +If you need a surface nobody has measured, add a check to `tools/wasmprobe/src/wasmprobe/checks.py` and run `just build-probe`. A check is a string of Python that leaves its answer in `result`, and it has to import everything it uses, because the check before it may have taken the runtime down. Mark it `TIER0` if a lesson would depend on it, and `just probe` will fail the build the day it stops working. If it already fails and you have decided what to do instead, write that decision in the check's `accepted` field rather than deleting the check, so the gap stays in the report and the next regression is still visible. + ## Definition of done for a lesson No partial credit on any of these. diff --git a/README.md b/README.md index 6f0e814..2ead907 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ Every chapter therefore produces two things. The chapter teaches, in prose and p **Every chapter runs in a browser.** Pyodide 314 is CPython 3.14 compiled to WebAssembly, so real bytecode, refcount, dictionary layout and garbage collection experiments work on a locked down school laptop with nothing installed. Chapters that need a debug build and a debugger ship a recorded session you can step through instead, generated in CI against a real build rather than captured by hand. +That claim was measured rather than assumed. Fifteen checks run on a native CPython and inside a real WebAssembly runtime, and twelve of them behave identically: `_testinternalcapi`, `ctypes` reading a live object header, `sys.monitoring`, `sys.settrace`, the cycle collector, and the whole front end. Threads cannot start, one call has to be made slightly differently, and one bad argument crashes the runtime instead of raising. The matrix, the raw runs and the decision that came out of them are in [probes/pyodide](probes/pyodide), and CI runs the same checks on every pull request so a Pyodide release that takes something away is noticed here rather than by a reader. + **CPython already contains three machine readable specifications of itself, and almost nobody teaches this.** `Grammar/python.gram` is the grammar. `Parser/Python.asdl` is the AST. `Python/bytecodes.c` is the interpreter semantics, written in a DSL that `Tools/cases_generator` compiles into the tier 1 interpreter, the tier 2 interpreter, the optimizer cases and every metadata table. CPython does not hand write its front end or its interpreter, it generates them. So the right architecture for a reimplementation is not to port the C, it is to add a backend, which is what the capstone does. **The compiler is already exposed to Python.** `_testinternalcapi` exports `compiler_codegen`, `optimize_cfg` and `assemble_code_object` on a stock interpreter. You can run the CPython compiler one stage at a time, from a notebook, and diff the control flow graph before and after optimization, with no build. This is the best teaching hook in the codebase and no existing course uses it. @@ -57,6 +59,7 @@ Pinned to `v3.15.0rc1` today and moving to `v3.15.0` when it ships on 1 October | `nbversion` | The lessons are written against 3.15 and every reader in Colab or in a browser widget is on 3.14. This runs all of them on both, compares the output cell by cell, and fails when a cell that differs has no note saying so, or carries a note that stopped being true | [tools/nbversion](tools/nbversion) | | `bpcheck` | The shape a blueprint has to have before somebody can implement from it: the nine sections in order, the header block, the invariant numbering, and no fact deferred to a lesson | [tools/bpcheck](tools/bpcheck) | | `bpc` | The blueprint compiler. Where upstream ships the material in a form a program can read, the specification is generated from it rather than typed. It reads `Parser/Python.asdl` with CPython's own parser and writes the three sections of BP-AST that list all 113 node kinds, each one citing the line it is declared on | [tools/bpc](tools/bpc) | +| `wasmprobe` | Asks a browser Python which of the surfaces the lessons depend on actually work, runs the same questions on a native interpreter for comparison, and fails the build when one of them stops working in the browser without a written decision about it | [tools/wasmprobe](tools/wasmprobe) | | `xraymanim` | The animations, and the fifteen shapes they are allowed to be made of. Each one is planned as a storyboard that is checked in milliseconds, so a mistake is caught before anybody pays for a render | [xraymanim](xraymanim) | | `xraywidgets` | The parts of a lesson you can click: a disassembler that shows what `dis` hides, a pipeline explorer with six panes from source to code object, and a prediction gate that asks before it tells. Each one renders twice from one piece of code: plain HTML with nothing installed, and the same picture with working buttons when anywidget is there | [xraywidgets](xraywidgets) | diff --git a/justfile b/justfile index 9ff9600..dc65fb2 100644 --- a/justfile +++ b/justfile @@ -27,7 +27,7 @@ vendor: git -C "{{cpython_src}}" rev-parse HEAD # The full local check, in the order that fails fastest. -check: lint test citations blueprints diagrams lessons notebooks animations +check: lint test citations blueprints diagrams lessons notebooks probe animations lint: uv run ruff check . @@ -130,6 +130,27 @@ animations: build-animations: uv run --extra anim xraymanim render +# Read the committed probe results and fail when a surface the lessons need has stopped +# working in the browser, or when the report and the notebook have fallen behind the checks. +# Milliseconds: it reads two JSON files, it does not boot anything. +probe: + uv run wasmprobe check probes/pyodide + uv run nbcheck run probes + +# Record the probe again on both runtimes and rewrite the report and the notebook. Needs +# node and `npm install` in tools/wasmprobe, and a 3.14 to compare against, because Pyodide +# ships 3.14 and a native 3.15 control would confuse a version difference for a build one. +# +# There is no `nbcheck lint probes` anywhere here on purpose. Those rules are written for a +# lesson: install pyxray, print the version banner, and so on. The probe installs nothing, +# which is the whole point of it, so it would fail two rules for doing its job properly. +build-probe: + UV_PROJECT_ENVIRONMENT=/tmp/venv-314 uv run --python 3.14 --all-packages \ + wasmprobe native --into probes/pyodide + uv run wasmprobe browser --into probes/pyodide + uv run wasmprobe report probes/pyodide --into probes/pyodide/report.md + uv run wasmprobe notebook --into probes/pyodide/probe.ipynb + # Rewrite the citation lockfile after a human has read the diff. This is deliberately # not part of `check`, because a checker that silently repairs itself checks nothing. recheck: diff --git a/probes/pyodide/decision.md b/probes/pyodide/decision.md new file mode 100644 index 0000000..716097a --- /dev/null +++ b/probes/pyodide/decision.md @@ -0,0 +1,61 @@ +# What stays in Tier 0 after measuring Pyodide + +This is the written half of gate Q1. The measured half is `report.md` next to it, and the raw runs are in `native.json` and `pyodide.json`. Reproduce them with `just build-probe`, or open `probe.ipynb` and run the same checks on whatever browser you are sitting in front of. + +## The short version + +Tier 0 survives. Fifteen checks, twelve behave the same in a browser as they do on a native CPython, and none of the three differences takes an experiment out of Tier 0. + +That is a better result than the issue expected. Rule 3 stands, and `pyxray.replay` stays a fallback in M2 rather than becoming a load bearing part of M0. + +## What was measured + +Pyodide 314.0.6 from npm, which is CPython 3.14.2 built for `emscripten-5.0.3-wasm32`, driven from Node. The control is a native CPython 3.14.7, deliberately 3.14 rather than the pinned 3.15, so a version difference does not get reported as a build difference. + +## The three differences + +**`optimize_cfg` cannot be called the way `pyxray` calls it.** `compiler_codegen` works and returns the same eight instructions in both places, but the metadata dictionary it hands back has no `consts` key in this build. It has `argcount`, `kwonlyargcount` and `posonlyargcount`, and that is all. Native 3.14 and 3.15 both include `consts`. Since `pyxray.compiler.stages` passes `metadata["consts"]` straight into `optimize_cfg`, that line raises `KeyError` in a browser. + +The optimizer itself is fine. Build a constants list of the right length from the instruction sequence and `optimize_cfg` runs and returns the same seven instructions it returns natively. So this is a missing key rather than a missing stage, and the fix is in our code, not in Pyodide. Filed as a bug. Until it lands, the compiler stage experiments in T05 stay in Tier 0 with that one line guarded. + +**A wrong constants list kills the runtime instead of raising.** Hand `optimize_cfg` a list that is too short and a native interpreter raises `ValueError: LOAD_CONST index 0 is out of range for consts (len=0)`. In WebAssembly the same call reads past the end of memory, the runtime does not come back, and in a notebook the kernel dies and the reader loses their work. + +This does not remove anything from Tier 0, but it does constrain how the pipeline widget is written. It has to build the constants list itself and never pass one it was given, because there is no way to catch this. The probe notebook runs that check last, on its own, with a paragraph warning the reader first. + +**A thread cannot be started.** `threading` imports and `threading.Thread(...)` constructs, and `start()` raises `RuntimeError: can't start new thread`. This is expected for a single threaded WebAssembly build without the pthread proxy, and it was already the assumption: the concurrency lessons are in M4 and were never Tier 0. Nothing moves. + +## What works, and is worth saying out loud + +`_testinternalcapi` imports, which was the check most likely to sink this. `compiler_codegen` and `optimize_cfg` are both there and both callable. + +`assemble_code_object` is present, and the probe only checks that it exists rather than calling it. That is not laziness. It asserts on its metadata instead of raising, and a failed assertion aborts the process, so calling it to see what happens would be the same class of mistake as the constants list above. `pyxray.compiler.assemble` refuses to call it for the same reason, tracked in issue 35. + +`ctypes` reads both fields in front of a live object: the reference count matches `sys.getrefcount`, and the type pointer one word further along really is `id(list)`. Object headers stay live in the browser rather than being shown from a recording. + +`sys.monitoring` registers a tool, sets a local event and fires the callback. The stepper stays in Tier 0. + +`sys.settrace` still reports call, line and return, so the fallback exists too. + +The cycle collector frees a two node cycle, has three generations, and is enabled. + +`dis`, `ast`, `symtable`, `tokenize`, `marshal`, `opcode` and `_opcode` all import, and `dis` gives the same five instructions, the same ten byte code object and the same 109 byte marshal blob as the native 3.14. + +## Where the two are genuinely different machines + +Four answers differ without anything being broken, and a lesson that asserts one of these numbers is teaching the build rather than the language. + +Pointers are four bytes rather than eight. `sysconfig.get_platform()` is `emscripten-5.0.3-wasm32`. The third garbage collector threshold is 0 rather than 10, so the oldest generation is never collected on a schedule. And the metadata key described above. + +The pointer size is the one to watch. Every diagram in the object lessons draws an eight byte word, and a reader in a browser who measures it gets four. T08 and T09 need a sentence about that, and it is a good sentence to have: it is the difference between memorising a number and knowing where the number comes from. + +## The phone question, answered partly + +The issue asks how long a cold boot takes on a mid range phone. This probe cannot answer that. It boots in about a second from a local disk under Node, which is a floor and not a promise. + +What it can measure is the part that dominates on a phone: 13.5 MB has to arrive before the first cell runs, which is the WebAssembly binary, the JavaScript glue, the standard library zip and the lock file. On a slow connection that is the wait, not the boot. Anything about tab memory, a service worker cache, or a real device is not answered here and is worth its own issue when the site actually exists. + +## What this changes + +Nothing moves from Tier 0 to Tier 1. + +Three pieces of work fall out of it. Guard the `metadata["consts"]` line in `pyxray.compiler.stages` so the compiler stages work in a browser. Make the pipeline widget build its own constants list rather than accepting one, because the alternative is a dead kernel. Add a sentence to the object lessons about the word size, and keep measuring it rather than asserting it, which is what those lessons already do for the small integer cache. diff --git a/probes/pyodide/native.json b/probes/pyodide/native.json new file mode 100644 index 0000000..b41a01c --- /dev/null +++ b/probes/pyodide/native.json @@ -0,0 +1,160 @@ +{ + "runtime": "native", + "python": "3.14.7", + "seconds": 0.0, + "payload_bytes": 0, + "outcomes": [ + { + "key": "version", + "status": "ok", + "value": { + "python": "3.14.7", + "platform": "macosx-15.0-arm64", + "pointer_bytes": 8, + "free_threaded": false + } + }, + { + "key": "internal_capi_import", + "status": "ok", + "value": { + "compiler_codegen": true, + "optimize_cfg": true, + "assemble_code_object": true + } + }, + { + "key": "compiler_codegen", + "status": "ok", + "value": { + "instructions": 8, + "first": 128, + "metadata_keys": [ + "argcount", + "consts", + "kwonlyargcount", + "posonlyargcount" + ] + } + }, + { + "key": "optimize_cfg", + "status": "ok", + "value": { + "instructions": 5 + } + }, + { + "key": "optimize_cfg_direct", + "status": "ok", + "value": { + "slots": 3, + "instructions": 7 + } + }, + { + "key": "optimize_cfg_short_consts", + "status": "ok", + "value": { + "raised": "ValueError: LOAD_CONST index 0 is out of range for consts (len=0)" + } + }, + { + "key": "ctypes_header", + "status": "ok", + "value": { + "refcount_field": 1, + "getrefcount": 2, + "type_pointer_matches": true, + "word_bytes": 8 + } + }, + { + "key": "monitoring", + "status": "ok", + "value": { + "fired": 1, + "event_names": 19 + } + }, + { + "key": "settrace", + "status": "ok", + "value": { + "events": [ + "call", + "line", + "return" + ] + } + }, + { + "key": "gc", + "status": "ok", + "value": { + "cycle_freed": true, + "thresholds": [ + 2000, + 10, + 10 + ], + "enabled": true, + "generations": 3 + } + }, + { + "key": "debugmallocstats", + "status": "ok", + "value": { + "callable": true + } + }, + { + "key": "front_end_modules", + "status": "ok", + "value": { + "dis": true, + "ast": true, + "symtable": true, + "tokenize": true, + "marshal": true, + "opcode": true, + "_opcode": true + } + }, + { + "key": "disassembly", + "status": "ok", + "value": { + "opnames": [ + "RESUME", + "LOAD_SMALL_INT", + "STORE_NAME", + "LOAD_CONST", + "RETURN_VALUE" + ], + "code_size": 10, + "consts": [ + "6", + "None" + ], + "marshal_size": 109 + } + }, + { + "key": "small_integers", + "status": "ok", + "value": { + "top": 256 + } + }, + { + "key": "threading", + "status": "ok", + "value": { + "ran": true, + "active": 1 + } + } + ] +} diff --git a/probes/pyodide/probe.ipynb b/probes/pyodide/probe.ipynb new file mode 100644 index 0000000..cbe335b --- /dev/null +++ b/probes/pyodide/probe.ipynb @@ -0,0 +1,355 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "probe-01", + "metadata": {}, + "source": [ + "# Which parts of CPython work in a browser\n", + "\n", + "This project promises that the first experiments in every lesson run in a browser tab with nothing installed. That rests on Pyodide, which is CPython compiled to WebAssembly, and on the introspection surfaces the lessons poke at surviving that build. Some of them do not, and the point of this notebook is to find out which, on the exact runtime you are sitting in front of rather than on one we tested a while ago.\n", + "\n", + "Run every cell. It takes a few seconds and installs nothing. At the end you get a table of what worked here, and you can compare it against the recordings committed next to this file.\n", + "\n", + "One warning worth reading first. One of the checks below asks what happens when the bytecode optimizer is handed a constants list that is too short. On a normal build that raises a tidy exception. In a WebAssembly build it reads past the end of memory and takes the whole runtime with it, which in a notebook means the kernel dies and you have to restart it. That check is last for exactly this reason, and everything above it will have already printed.\n", + "\n", + "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tamnd/cpython-internals/blob/main/probes/pyodide/probe.ipynb)" + ] + }, + { + "cell_type": "markdown", + "id": "probe-02", + "metadata": {}, + "source": [ + "## The checks\n", + "\n", + "Each one is a small piece of Python that leaves its answer in a variable called `result`. They are written out in full rather than imported, so you can read what is being asked, and edit one to ask something else." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "probe-03", + "metadata": {}, + "outputs": [], + "source": [ + "CHECKS = {\n", + " # Which CPython is this, and what was it built for\n", + " \"version\": \"\"\"\n", + "import platform\n", + "import sys\n", + "import sysconfig\n", + "\n", + "result = {\n", + " \"python\": platform.python_version(),\n", + " \"platform\": sysconfig.get_platform(),\n", + " \"pointer_bytes\": sys.maxsize.bit_length() // 8 + 1,\n", + " \"free_threaded\": bool(sysconfig.get_config_var(\"Py_GIL_DISABLED\")),\n", + "}\n", + "\"\"\",\n", + " # Is _testinternalcapi importable at all\n", + " \"internal_capi_import\": \"\"\"\n", + "import _testinternalcapi\n", + "\n", + "wanted = (\"compiler_codegen\", \"optimize_cfg\", \"assemble_code_object\")\n", + "result = {name: hasattr(_testinternalcapi, name) for name in wanted}\n", + "\"\"\",\n", + " # Does compiler_codegen turn a tree into an instruction sequence\n", + " \"compiler_codegen\": \"\"\"\n", + "import _testinternalcapi\n", + "import ast\n", + "\n", + "sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse(\"answer = 6 * 7\"), \"\", 0)\n", + "instructions = sequence.get_instructions()\n", + "result = {\n", + " \"instructions\": len(instructions),\n", + " \"first\": instructions[0][0],\n", + " \"metadata_keys\": sorted(metadata),\n", + "}\n", + "\"\"\",\n", + " # Does optimize_cfg run over that sequence the way pyxray calls it\n", + " \"optimize_cfg\": \"\"\"\n", + "import _testinternalcapi\n", + "import ast\n", + "\n", + "sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse(\"answer = 6 * 7\"), \"\", 0)\n", + "optimized = _testinternalcapi.optimize_cfg(sequence, metadata[\"consts\"], 0)\n", + "result = {\"instructions\": len(optimized.get_instructions())}\n", + "\"\"\",\n", + " # Does optimize_cfg run at all, given a constants list built by hand\n", + " \"optimize_cfg_direct\": \"\"\"\n", + "import _testinternalcapi\n", + "import ast\n", + "import opcode\n", + "\n", + "sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse(\"answer = 6 * 7\"), \"\", 0)\n", + "load = opcode.opmap[\"LOAD_CONST\"]\n", + "slots = [one[1] for one in sequence.get_instructions() if one[0] == load]\n", + "consts = [None] * (max(slots) + 1 if slots else 0)\n", + "optimized = _testinternalcapi.optimize_cfg(sequence, consts, 0)\n", + "result = {\"slots\": len(consts), \"instructions\": len(optimized.get_instructions())}\n", + "\"\"\",\n", + " # Can ctypes read the two fields in front of every object\n", + " \"ctypes_header\": \"\"\"\n", + "import ctypes\n", + "import sys\n", + "\n", + "value = [1, 2, 3]\n", + "word = ctypes.sizeof(ctypes.c_ssize_t)\n", + "result = {\n", + " \"refcount_field\": ctypes.c_ssize_t.from_address(id(value)).value,\n", + " \"getrefcount\": sys.getrefcount(value),\n", + " \"type_pointer_matches\": ctypes.c_void_p.from_address(id(value) + word).value == id(list),\n", + " \"word_bytes\": word,\n", + "}\n", + "\"\"\",\n", + " # Does sys.monitoring register a callback and fire it\n", + " \"monitoring\": \"\"\"\n", + "import sys\n", + "\n", + "TOOL = 5\n", + "seen = []\n", + "sys.monitoring.use_tool_id(TOOL, \"wasmprobe\")\n", + "try:\n", + " def target():\n", + " return 1 + 1\n", + " sys.monitoring.register_callback(\n", + " TOOL, sys.monitoring.events.PY_START, lambda *arguments: seen.append(arguments)\n", + " )\n", + " sys.monitoring.set_local_events(TOOL, target.__code__, sys.monitoring.events.PY_START)\n", + " target()\n", + "finally:\n", + " sys.monitoring.free_tool_id(TOOL)\n", + "allowed = [\n", + " name\n", + " for name in dir(sys.monitoring.events)\n", + " if name.isupper() and not name.startswith(\"NO_\")\n", + "]\n", + "result = {\"fired\": len(seen), \"event_names\": len(allowed)}\n", + "\"\"\",\n", + " # Does sys.settrace still see call, line and return\n", + " \"settrace\": \"\"\"\n", + "import sys\n", + "\n", + "seen = []\n", + "def tracer(frame, event, argument):\n", + " seen.append(event)\n", + " return tracer\n", + "def target():\n", + " return 2\n", + "sys.settrace(tracer)\n", + "try:\n", + " target()\n", + "finally:\n", + " sys.settrace(None)\n", + "result = {\"events\": seen}\n", + "\"\"\",\n", + " # Does the cycle collector behave the way T09 says it does\n", + " \"gc\": \"\"\"\n", + "import gc\n", + "import weakref\n", + "\n", + "class Node:\n", + " pass\n", + "first, second = Node(), Node()\n", + "watch = weakref.ref(first)\n", + "first.other, second.other = second, first\n", + "del first, second\n", + "gc.collect()\n", + "# What gc.collect() returns counts everything it swept, including whatever else this\n", + "# process happened to be holding, so it is different every time. Whether this particular\n", + "# cycle went away is the question the lesson actually asks.\n", + "result = {\n", + " \"cycle_freed\": watch() is None,\n", + " \"thresholds\": list(gc.get_threshold()),\n", + " \"enabled\": gc.isenabled(),\n", + " \"generations\": len(gc.get_stats()),\n", + "}\n", + "\"\"\",\n", + " # Does sys._debugmallocstats produce anything under Emscripten's allocator\n", + " \"debugmallocstats\": \"\"\"\n", + "import sys\n", + "\n", + "result = {\"callable\": callable(getattr(sys, \"_debugmallocstats\", None))}\n", + "\"\"\",\n", + " # Do dis, ast, symtable, tokenize and marshal all import\n", + " \"front_end_modules\": \"\"\"\n", + "found = {}\n", + "for name in (\"dis\", \"ast\", \"symtable\", \"tokenize\", \"marshal\", \"opcode\", \"_opcode\"):\n", + " try:\n", + " __import__(name)\n", + " found[name] = True\n", + " except Exception as error:\n", + " found[name] = f\"{type(error).__name__}: {error}\"\n", + "result = found\n", + "\"\"\",\n", + " # Does dis give the same instructions as a native interpreter\n", + " \"disassembly\": \"\"\"\n", + "import dis\n", + "import marshal\n", + "\n", + "source = \"answer = 6 * 7\"\n", + "code = compile(source, \"\", \"exec\")\n", + "result = {\n", + " \"opnames\": [one.opname for one in dis.get_instructions(source)],\n", + " \"code_size\": len(code.co_code),\n", + " \"consts\": [repr(one) for one in code.co_consts],\n", + " \"marshal_size\": len(marshal.dumps(code)),\n", + "}\n", + "\"\"\",\n", + " # Where does the shared range of small integers stop\n", + " \"small_integers\": \"\"\"\n", + "top = 0\n", + "for candidate in range(0, 4096):\n", + " if int(str(candidate)) is int(str(candidate)):\n", + " top = candidate\n", + "result = {\"top\": top}\n", + "\"\"\",\n", + " # Can a thread be started\n", + " \"threading\": \"\"\"\n", + "import threading\n", + "\n", + "ran = []\n", + "worker = threading.Thread(target=lambda: ran.append(True))\n", + "worker.start()\n", + "worker.join()\n", + "result = {\"ran\": ran == [True], \"active\": threading.active_count()}\n", + "\"\"\",\n", + "}\n", + "\n", + "print(f\"{len(CHECKS)} checks\")" + ] + }, + { + "cell_type": "markdown", + "id": "probe-04", + "metadata": {}, + "source": [ + "## Running them\n", + "\n", + "Each check gets a fresh namespace and anything it throws is caught and recorded, so one failure does not stop the rest. A check that takes the runtime down cannot be caught, which is why the dangerous one is separated out below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "probe-05", + "metadata": {}, + "outputs": [], + "source": [ + "answers = {}\n", + "for key, source in CHECKS.items():\n", + " namespace = {}\n", + " try:\n", + " exec(source, namespace)\n", + " except BaseException as error:\n", + " answers[key] = (\"raised\", f\"{type(error).__name__}: {error}\")\n", + " else:\n", + " answers[key] = (\"ok\", namespace.get(\"result\"))\n", + "\n", + "print(f\"{sum(1 for status, _ in answers.values() if status == 'ok')} of {len(answers)} worked\")" + ] + }, + { + "cell_type": "markdown", + "id": "probe-06", + "metadata": {}, + "source": [ + "### The table\n", + "\n", + "The same shape as the matrix in `report.md` next to this notebook, so the two are easy to read against each other. The answer column is whatever the check left in `result`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "probe-07", + "metadata": {}, + "outputs": [], + "source": [ + "def matrix(answers):\n", + " \"\"\"Print what happened, one row per check.\"\"\"\n", + " width = max(len(key) for key in answers)\n", + " print(f\"{'check'.ljust(width)} status answer\")\n", + " print(\"-\" * (width + 40))\n", + " for key, (status, answer) in answers.items():\n", + " print(f\"{key.ljust(width)} {status.ljust(6)} {answer}\")\n", + "\n", + "\n", + "matrix(answers)" + ] + }, + { + "cell_type": "markdown", + "id": "probe-08", + "metadata": {}, + "source": [ + "## The one that can kill the kernel\n", + "\n", + "Everything above has already printed, so run this last. The cell catches the exception itself, so on a normal build you get a tidy `ValueError` naming the constant it could not find. In a browser there is nothing to catch: the read goes past the end of memory, the runtime does not come back, and you restart the kernel.\n", + "\n", + "That difference is the reason the pipeline widget in the lessons builds its own constants list rather than trusting the one it is handed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "probe-09", + "metadata": {}, + "outputs": [], + "source": [ + "import _testinternalcapi\n", + "import ast\n", + "\n", + "sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse(\"answer = 6 * 7\"), \"\", 0)\n", + "try:\n", + " _testinternalcapi.optimize_cfg(sequence, [], 0)\n", + " result = {\"raised\": None}\n", + "except BaseException as error:\n", + " result = {\"raised\": f\"{type(error).__name__}: {error}\"}" + ] + }, + { + "cell_type": "markdown", + "id": "probe-10", + "metadata": {}, + "source": [ + "If the cell above came back rather than killing the kernel, here is what it caught. If the kernel died, that is the answer, and it is the one worth telling us about." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "probe-11", + "metadata": {}, + "outputs": [], + "source": [ + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "probe-12", + "metadata": {}, + "source": [ + "## What to do with this\n", + "\n", + "Compare what you got against `report.md` in this directory, which is the same matrix recorded on a native CPython and on Pyodide under Node.\n", + "\n", + "A check that failed here and passed there is usually your environment: an old Pyodide, a sandbox that blocks threads, a Python built without the test modules. A check that failed in both is our problem, and worth an issue." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/probes/pyodide/pyodide.json b/probes/pyodide/pyodide.json new file mode 100644 index 0000000..ef223bd --- /dev/null +++ b/probes/pyodide/pyodide.json @@ -0,0 +1,152 @@ +{ + "runtime": "pyodide", + "python": "3.14.2", + "seconds": 0.9, + "payload_bytes": 13508566, + "outcomes": [ + { + "key": "version", + "status": "ok", + "value": { + "python": "3.14.2", + "platform": "emscripten-5.0.3-wasm32", + "pointer_bytes": 4, + "free_threaded": false + } + }, + { + "key": "internal_capi_import", + "status": "ok", + "value": { + "compiler_codegen": true, + "optimize_cfg": true, + "assemble_code_object": true + } + }, + { + "key": "compiler_codegen", + "status": "ok", + "value": { + "instructions": 8, + "first": 128, + "metadata_keys": [ + "argcount", + "kwonlyargcount", + "posonlyargcount" + ] + } + }, + { + "key": "optimize_cfg", + "status": "raised", + "error": "KeyError: 'consts'" + }, + { + "key": "optimize_cfg_direct", + "status": "ok", + "value": { + "slots": 3, + "instructions": 7 + } + }, + { + "key": "optimize_cfg_short_consts", + "status": "fatal", + "error": "memory access out of bounds" + }, + { + "key": "ctypes_header", + "status": "ok", + "value": { + "refcount_field": 1, + "getrefcount": 2, + "type_pointer_matches": true, + "word_bytes": 4 + } + }, + { + "key": "monitoring", + "status": "ok", + "value": { + "fired": 1, + "event_names": 19 + } + }, + { + "key": "settrace", + "status": "ok", + "value": { + "events": [ + "call", + "line", + "return" + ] + } + }, + { + "key": "gc", + "status": "ok", + "value": { + "cycle_freed": true, + "thresholds": [ + 2000, + 10, + 0 + ], + "enabled": true, + "generations": 3 + } + }, + { + "key": "debugmallocstats", + "status": "ok", + "value": { + "callable": true + } + }, + { + "key": "front_end_modules", + "status": "ok", + "value": { + "dis": true, + "ast": true, + "symtable": true, + "tokenize": true, + "marshal": true, + "opcode": true, + "_opcode": true + } + }, + { + "key": "disassembly", + "status": "ok", + "value": { + "opnames": [ + "RESUME", + "LOAD_SMALL_INT", + "STORE_NAME", + "LOAD_CONST", + "RETURN_VALUE" + ], + "code_size": 10, + "consts": [ + "6", + "None" + ], + "marshal_size": 109 + } + }, + { + "key": "small_integers", + "status": "ok", + "value": { + "top": 256 + } + }, + { + "key": "threading", + "status": "raised", + "error": "RuntimeError: can't start new thread" + } + ] +} diff --git a/probes/pyodide/report.md b/probes/pyodide/report.md new file mode 100644 index 0000000..c7dab06 --- /dev/null +++ b/probes/pyodide/report.md @@ -0,0 +1,70 @@ +# What works under Pyodide + +Generated by `wasmprobe report`. Native run on CPython 3.14.7, WebAssembly run on Pyodide with CPython 3.14.2, which booted in 0.9 seconds off a local disk after 13.5 MB of runtime and standard library. + +15 checks: 12 works in both, 3 works natively, not in the browser. + +| Check | Needed for | 3.14.7 native | 3.14.2 WebAssembly | +| --- | --- | --- | --- | +| Which CPython is this, and what was it built for | info | yes | yes | +| Is _testinternalcapi importable at all | tier0 | yes | yes | +| Does compiler_codegen turn a tree into an instruction sequence | tier0 | yes | yes | +| Does optimize_cfg run over that sequence the way pyxray calls it | tier0, known gap | yes | raises, KeyError: 'consts' | +| Does optimize_cfg run at all, given a constants list built by hand | tier0 | yes | yes | +| What happens when optimize_cfg is handed a constants list that is too short | info | yes | kills the runtime, memory access out of bounds | +| Can ctypes read the two fields in front of every object | tier0 | yes | yes | +| Does sys.monitoring register a callback and fire it | tier0 | yes | yes | +| Does sys.settrace still see call, line and return | nice | yes | yes | +| Does the cycle collector behave the way T09 says it does | tier0 | yes | yes | +| Does sys._debugmallocstats produce anything under Emscripten's allocator | nice | yes | yes | +| Do dis, ast, symtable, tokenize and marshal all import | tier0 | yes | yes | +| Does dis give the same instructions as a native interpreter | tier0 | yes | yes | +| Where does the shared range of small integers stop | info | yes | yes | +| Can a thread be started | nice | yes | raises, RuntimeError: can't start new thread | + +## What a failure would cost + +**Which CPython is this, and what was it built for.** Nothing. Every other answer here is about this interpreter. + +**Is _testinternalcapi importable at all.** Everything that shows the compiler one stage at a time. It is the single best hook this project has, and without it T05 has nothing live to show. + +**Does compiler_codegen turn a tree into an instruction sequence.** The first compiler stage in T05 and in the pipeline explorer widget. + +**Does optimize_cfg run over that sequence the way pyxray calls it.** The middle stage. It is the one that makes 6 * 7 disappear, which is the single most convincing thing in the first part of the course. + +**Does optimize_cfg run at all, given a constants list built by hand.** The same middle stage, asked in a way a missing metadata key cannot hide. + +**What happens when optimize_cfg is handed a constants list that is too short.** Nothing, and it is worth knowing. A clean exception means the widget can show the reader their mistake. A dead runtime means it has to check first. + +**Can ctypes read the two fields in front of every object.** Object headers shown from a recording rather than from the reader's own live objects. A real loss and survivable. + +**Does sys.monitoring register a callback and fire it.** The stepper goes offline for Tier 0, and watching a function run one instruction at a time is most of T07. + +**Does sys.settrace still see call, line and return.** The fallback for anything sys.monitoring cannot do. Losing both would be the bad case. + +**Does the cycle collector behave the way T09 says it does.** T09 shows a cycle being collected. If the numbers are different in the browser the lesson is teaching Emscripten rather than CPython. + +**Does sys._debugmallocstats produce anything under Emscripten's allocator.** One cell in T09. It writes to the real standard error rather than to sys.stderr, so a notebook cannot capture it anyway. + +**Do dis, ast, symtable, tokenize and marshal all import.** Most of the first part of the course. These are pure Python or small C modules, so a failure here would be a surprise. + +**Does dis give the same instructions as a native interpreter.** Every bytecode listing in the course. A difference here is a difference in the compiled build rather than in the language. + +**Where does the shared range of small integers stop.** Nothing, and T08 measures it rather than asserting it for this reason. + +**Can a thread be started.** The whole concurrency part in the browser. Those lessons are late enough that a recording is an acceptable answer, and it has to be a deliberate one. + +## Where the two answers differ + +Both runtimes answered these, and answered them differently. + +| Check | Native | WebAssembly | +| --- | --- | --- | +| Which CPython is this, and what was it built for | `{'python': '3.14.7', 'platform': 'macosx-15.0-arm64', 'pointer_bytes': 8, 'free_threaded': False}` | `{'python': '3.14.2', 'platform': 'emscripten-5.0.3-wasm32', 'pointer_bytes': 4, 'free_threaded': False}` | +| Does compiler_codegen turn a tree into an instruction sequence | `{'instructions': 8, 'first': 128, 'metadata_keys': ['argcount', 'consts', 'kwonlyargcount', 'posonlyargcount']}` | `{'instructions': 8, 'first': 128, 'metadata_keys': ['argcount', 'kwonlyargcount', 'posonlyargcount']}` | +| Can ctypes read the two fields in front of every object | `{'refcount_field': 1, 'getrefcount': 2, 'type_pointer_matches': True, 'word_bytes': 8}` | `{'refcount_field': 1, 'getrefcount': 2, 'type_pointer_matches': True, 'word_bytes': 4}` | +| Does the cycle collector behave the way T09 says it does | `{'cycle_freed': True, 'thresholds': [2000, 10, 10], 'enabled': True, 'generations': 3}` | `{'cycle_freed': True, 'thresholds': [2000, 10, 0], 'enabled': True, 'generations': 3}` | + +## Known gaps, and what we do instead + +**Does optimize_cfg run over that sequence the way pyxray calls it.** The metadata this build hands back has no consts key, so the call fails before it reaches the optimizer. The check below shows the optimizer itself is fine, so in the browser the pipeline builds that list from the instruction sequence instead of asking for it. diff --git a/pyproject.toml b/pyproject.toml index 6db043e..0c696f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "nbversion", "pyxray", "refcheck", + "wasmprobe", "xraymanim", "xraywidgets", ] @@ -45,6 +46,7 @@ members = [ "tools/nbdiagram", "tools/nbversion", "tools/refcheck", + "tools/wasmprobe", "xraymanim", "xraywidgets", ] @@ -58,6 +60,7 @@ nbdiagram = { workspace = true } nbversion = { workspace = true } pyxray = { workspace = true } refcheck = { workspace = true } +wasmprobe = { workspace = true } xraymanim = { workspace = true } xraywidgets = { workspace = true } @@ -71,6 +74,7 @@ testpaths = [ "tools/nbdiagram/tests", "tools/nbversion/tests", "tools/refcheck/tests", + "tools/wasmprobe/tests", "xraymanim/tests", "xraywidgets/tests", ] @@ -78,6 +82,7 @@ addopts = "-q --strict-markers --strict-config" markers = [ "pinned: needs the pinned CPython source tree", "freethreaded: needs a free threaded build", + "slow: boots a WebAssembly runtime, so it takes seconds rather than milliseconds", ] filterwarnings = [ "error", @@ -132,6 +137,8 @@ ignore = [ # The shared version notes are prose too, one sentence per line, and they are read by a # person in a notebook rather than in the source. Same reason as the lesson builders. "tools/nbbuild/src/nbbuild/notes.py" = ["E501"] +# The probe notebook's markdown, same rule again. It is read in a browser, not here. +"tools/wasmprobe/src/wasmprobe/notebook.py" = ["E501"] # Every word a widget shows a reader lives in one dictionary, written one sentence per line. # Splitting a sentence across two source lines to please a line length rule would leave the # sentence looking wrapped in the source and identical on screen, which helps nobody. diff --git a/tools/wasmprobe/README.md b/tools/wasmprobe/README.md new file mode 100644 index 0000000..7b97ce5 --- /dev/null +++ b/tools/wasmprobe/README.md @@ -0,0 +1,37 @@ +# wasmprobe + +Asks a browser Python which of the surfaces this project needs actually work, and puts the answer next to the same question asked on a normal interpreter. + +Every Tier 0 experiment in the lessons is meant to run in a browser tab with nothing installed. That rests on Pyodide, which is CPython compiled to WebAssembly by Emscripten, and on things like `_testinternalcapi`, `ctypes`, `sys.monitoring` and the cycle collector surviving that build. Most of them do. Not all of them, and the ones that do not are worth knowing about before a reader finds them. + +## Running it + +``` +just build-probe +``` + +That records the checks twice and rewrites the report and the notebook. It needs `node` on PATH and `npm install` run once inside this directory, and it needs a 3.14 to use as the control, because Pyodide ships 3.14 and comparing it against a native 3.15 would report version differences as build differences. + +``` +just probe +``` + +That is the fast half, and the one in `just check`. It reads the two committed recordings, fails when a check the lessons depend on works natively and not in the browser, fails when the report or the notebook has fallen behind the checks, and then runs the notebook to make sure it still works. + +## What is in here + +`checks.py` is the list of questions. Each one is a string of Python rather than a function, because the same source has to run in two places and shipping the source is the only way to stop the two copies drifting. A check leaves its answer in a variable called `result`, must not print, and must not depend on any earlier check, because a check that takes the runtime down leaves the next one a brand new one. + +Each check carries a weight. `tier0` means a lesson depends on it and a failure moves that experiment to Tier 1. `nice` means we want it and can live without it. `info` means the number is worth recording and nothing hangs on it. A `tier0` check can also carry an `accepted` sentence, which says we already know it fails in the browser and what we do instead. That keeps a known gap visible in the report without leaving the build permanently red, so the next regression is still noticeable. + +`driver.mjs` is the Node side. It boots Pyodide, runs the checks one at a time, and when one takes the whole runtime down it records that against the right check and boots a fresh runtime for the next one. That is not hypothetical: handing `optimize_cfg` a constants list that is too short reads past the end of memory in this build, where a native interpreter raises a tidy `ValueError`. + +`browser.py` drives that script and fills in a `skipped` outcome for anything the driver never reached. `native.py` runs the same checks here. `report.py` turns the two into the matrix. `notebook.py` writes the checks out as a notebook a reader can open in Colab or JupyterLite and run on their own runtime, which is the version of this that does not ask anybody to trust our recording. + +## The results + +They live in `probes/pyodide` at the top of the repository: `native.json`, `pyodide.json`, the rendered `report.md`, and `probe.ipynb`. + +## Node, not a real browser + +The WebAssembly is the same either way, and a headless browser in CI is another thing to keep working. What this does miss is the browser's own limits, so the boot time in the report is a floor rather than a promise, and anything about tab memory or a service worker is not answered here. The notebook exists to cover that gap: run it where you actually are. diff --git a/tools/wasmprobe/driver.mjs b/tools/wasmprobe/driver.mjs new file mode 100644 index 0000000..298454a --- /dev/null +++ b/tools/wasmprobe/driver.mjs @@ -0,0 +1,117 @@ +// Runs the checks inside Pyodide, which is CPython built for WebAssembly. +// +// Node rather than a real browser because the WebAssembly is the same either way and a +// headless browser in CI is a second thing to keep working. The one thing this misses is +// the browser's own limits, which is why the boot time here is a floor rather than a +// promise. +// +// The awkward part, and the reason this is not four lines. A check can take the whole +// runtime down: `optimize_cfg` in the build shipping today does, with a memory access out +// of bounds, and after that every later check would look broken. So the checks run one at +// a time, a fatal error is recorded against the check that caused it, and the next check +// starts in a runtime that was booted fresh. +// +// node driver.mjs checks.json out.json + +import { readFileSync, statSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { loadPyodide } from "pyodide"; + +const [checksPath, outPath] = process.argv.slice(2); +if (!checksPath || !outPath) { + console.error("usage: node driver.mjs checks.json out.json"); + process.exit(2); +} + +const checks = JSON.parse(readFileSync(checksPath, "utf8")); +const outcomes = []; +let bootSeconds = 0; +let python = "unknown"; + +// What a browser has to fetch before the first cell can run. Node reads these off the +// disk, so the boot time above is far better than a real one, and this number is the part +// that decides whether somebody on a phone waits or closes the tab. +function payloadBytes() { + const require = createRequire(import.meta.url); + const home = dirname(require.resolve("pyodide")); + const needed = ["pyodide.asm.wasm", "pyodide.asm.mjs", "python_stdlib.zip", "pyodide-lock.json"]; + let total = 0; + for (const name of needed) { + try { + total += statSync(join(home, name)).size; + } catch { + // A file that moved between Pyodide releases. Better an undercount than a crash. + } + } + return total; +} + +// The check's source leaves its answer in `result`. Doing the JSON encoding inside Python +// rather than handing the object across means a value Pyodide cannot convert, and there +// are several, shows up as a check that raised rather than as a driver that crashed. +const wrap = (source) => ` +import json, traceback +__wasmprobe__ = {} +try: + exec(${JSON.stringify(source)}, __wasmprobe__) + __out__ = json.dumps({"status": "ok", "value": __wasmprobe__.get("result")}, default=str) +except BaseException: + __line__ = traceback.format_exc().strip().splitlines()[-1] + __out__ = json.dumps({"status": "raised", "error": __line__}) +__out__ +`; + +async function boot() { + const started = performance.now(); + const runtime = await loadPyodide({ stdout: () => {}, stderr: () => {} }); + const seconds = (performance.now() - started) / 1000; + return { runtime, seconds }; +} + +let session = await boot(); +bootSeconds = session.seconds; +python = session.runtime.runPython("import platform; platform.python_version()"); + +const save = () => + writeFileSync( + outPath, + JSON.stringify( + { + runtime: "pyodide", + python, + seconds: Number(bootSeconds.toFixed(3)), + payload_bytes: payloadBytes(), + outcomes, + }, + null, + 2, + ) + "\n", + "utf8", + ); + +for (const check of checks) { + // Written before the check runs, so a crash that takes the process out as well as the + // runtime still leaves the file naming the check that did it. + outcomes.push({ key: check.key, status: "fatal", error: "the runtime did not come back" }); + save(); + let answer; + try { + answer = JSON.parse(session.runtime.runPython(wrap(check.source))); + } catch (error) { + // A Python exception is caught inside Python above, so anything arriving here has + // taken the runtime with it and the next check needs a new one. + outcomes[outcomes.length - 1] = { + key: check.key, + status: "fatal", + error: String(error.message || error).split("\n")[0], + }; + save(); + session = await boot(); + continue; + } + outcomes[outcomes.length - 1] = { key: check.key, ...answer }; + save(); +} + +save(); diff --git a/tools/wasmprobe/package-lock.json b/tools/wasmprobe/package-lock.json new file mode 100644 index 0000000..8468a20 --- /dev/null +++ b/tools/wasmprobe/package-lock.json @@ -0,0 +1,55 @@ +{ + "name": "wasmprobe-driver", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "wasmprobe-driver", + "version": "0.1.0", + "dependencies": { + "pyodide": "314.0.6" + } + }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "license": "MIT" + }, + "node_modules/pyodide": { + "version": "314.0.6", + "resolved": "https://registry.npmjs.org/pyodide/-/pyodide-314.0.6.tgz", + "integrity": "sha512-BKDTJyIqFxC4BExLqeRS3f5xvXZIjOt8C3zGLN/Cc7tFxSwvKVhVkchQQ2AGLOtR4YrVOIFxbV8poyDOOmWwxQ==", + "license": "MPL-2.0", + "dependencies": { + "@types/emscripten": "^1.41.4", + "ws": "^8.5.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/wasmprobe/package.json b/tools/wasmprobe/package.json new file mode 100644 index 0000000..0aa0d09 --- /dev/null +++ b/tools/wasmprobe/package.json @@ -0,0 +1,13 @@ +{ + "name": "wasmprobe-driver", + "version": "0.1.0", + "private": true, + "description": "Boots Pyodide and runs the wasmprobe checks inside it", + "type": "module", + "scripts": { + "probe": "node driver.mjs" + }, + "dependencies": { + "pyodide": "314.0.6" + } +} diff --git a/tools/wasmprobe/pyproject.toml b/tools/wasmprobe/pyproject.toml new file mode 100644 index 0000000..5b6fa0c --- /dev/null +++ b/tools/wasmprobe/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "wasmprobe" +version = "0.1.0" +description = "Ask a browser Python which of the introspection surfaces this project needs actually work" +requires-python = ">=3.14" +license = "MIT" +dependencies = [] + +[project.scripts] +wasmprobe = "wasmprobe.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/wasmprobe"] diff --git a/tools/wasmprobe/src/wasmprobe/__init__.py b/tools/wasmprobe/src/wasmprobe/__init__.py new file mode 100644 index 0000000..492b0d0 --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/__init__.py @@ -0,0 +1,36 @@ +"""Ask a browser Python which of the surfaces this project needs actually work. + +Every Tier 0 experiment in the lessons is supposed to run in a browser tab with nothing +installed. That rests on Pyodide, which is CPython compiled to WebAssembly, and on the +introspection surfaces the lessons poke at surviving that build. This package runs the same +list of checks twice, once here and once there, and puts the two answers side by side. + +The answer is not all good news, which is the point of measuring rather than assuming. +""" + +from .browser import Missing +from .checks import BY_KEY, CHECKS, INFO, NICE, TIER0, Check +from .report import differences, markdown, regressions, summary, table, verdict +from .result import FATAL, OK, RAISED, SKIPPED, Outcome, Run + +__all__ = [ + "BY_KEY", + "CHECKS", + "FATAL", + "INFO", + "NICE", + "OK", + "RAISED", + "SKIPPED", + "TIER0", + "Check", + "Missing", + "Outcome", + "Run", + "differences", + "markdown", + "regressions", + "summary", + "table", + "verdict", +] diff --git a/tools/wasmprobe/src/wasmprobe/browser.py b/tools/wasmprobe/src/wasmprobe/browser.py new file mode 100644 index 0000000..3b9b78d --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/browser.py @@ -0,0 +1,87 @@ +"""Running the same checks on Pyodide, which is CPython built for WebAssembly. + +Python cannot host that runtime itself, so this hands the work to a small Node script and +reads back what it wrote. The checks travel as JSON so the Node side never has to know +anything about this package beyond "here is a list of things with a source field". + +The one thing this module knows that the driver does not is what a missing outcome means. +A check the driver never reached, because an earlier one took the process down rather than +just the runtime, comes back as skipped rather than as a silent absence. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import tempfile +from pathlib import Path + +from .checks import CHECKS, Check +from .result import FATAL, SKIPPED, Outcome, Run + +#: The driver, next to the package rather than inside it, because it is not Python. +DRIVER = Path(__file__).resolve().parents[2] / "driver.mjs" + + +class Missing(RuntimeError): + """Node or the Pyodide package is not installed.""" + + +def ready() -> str: + """The problem stopping a browser run, or an empty string when there is none.""" + if shutil.which("node") is None: + return "node is not on PATH" + if not (DRIVER.parent / "node_modules" / "pyodide").is_dir(): + return f"pyodide is not installed, run npm install in {DRIVER.parent}" + return "" + + +def run(checks: list[Check] | None = None, timeout: int = 900) -> Run: + """Every check, in order, inside a WebAssembly runtime.""" + problem = ready() + if problem: + raise Missing(problem) + wanted = checks if checks is not None else CHECKS + with tempfile.TemporaryDirectory() as room: + inbox = Path(room) / "checks.json" + outbox = Path(room) / "out.json" + inbox.write_text( + json.dumps([{"key": one.key, "source": one.source} for one in wanted]), + encoding="utf-8", + ) + finished = subprocess.run( + ["node", str(DRIVER), str(inbox), str(outbox)], + cwd=DRIVER.parent, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + if not outbox.exists(): + tail = (finished.stderr or finished.stdout).strip().splitlines() + raise Missing("the driver wrote nothing: " + (tail[-1] if tail else "no output")) + body = json.loads(outbox.read_text(encoding="utf-8")) + return _assemble(body, wanted, finished.returncode) + + +def _assemble(body: dict, wanted: list[Check], code: int) -> Run: + """Turn the driver's JSON into a Run, filling in the checks it never reached.""" + outcomes = {one["key"]: Outcome.from_dict(one) for one in body["outcomes"]} + for check in wanted: + if check.key not in outcomes: + outcomes[check.key] = Outcome(check.key, SKIPPED, error="the run stopped first") + if code != 0: + # The driver exiting badly means the last check it started took the process with + # it, not only the runtime. The placeholder it wrote before starting is already + # fatal, so this only fills in a better sentence. + last = body["outcomes"][-1]["key"] if body["outcomes"] else "" + if last and outcomes[last].status == FATAL: + outcomes[last] = Outcome(last, FATAL, error="took the whole process down") + return Run( + runtime=str(body.get("runtime", "pyodide")), + python=str(body.get("python", "unknown")), + outcomes={check.key: outcomes[check.key] for check in wanted}, + seconds=float(body.get("seconds", 0.0)), + payload_bytes=int(body.get("payload_bytes", 0)), + ) diff --git a/tools/wasmprobe/src/wasmprobe/checks.py b/tools/wasmprobe/src/wasmprobe/checks.py new file mode 100644 index 0000000..a690907 --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/checks.py @@ -0,0 +1,350 @@ +"""The questions this project needs a browser Python to answer. + +Every Tier 0 experiment in the lessons runs on Pyodide, which is CPython compiled to +WebAssembly by Emscripten. The promise is that somebody on a locked down laptop can do +real internals work with nothing installed, and that promise rests entirely on which +introspection surfaces survive that build. Nobody had checked. + +Each check is a string of Python rather than a function, because the same source has to run +in two places: in this process, on a native interpreter, and inside a WebAssembly runtime +driven from Node. Shipping source means the two runs cannot drift apart, which they would +the first time somebody edited one and forgot the other. + +A check returns something JSON can carry. It must not print, and it must not depend on any +earlier check having run, because a check that takes the runtime down with it means +everything after it starts from a fresh one. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +#: A Tier 0 experiment somewhere in the lessons needs this. If it fails in the browser, +#: that experiment has to move to Tier 1 and be shown from a recording instead. +TIER0 = "tier0" + +#: Wanted, and survivable. A failure costs a nicer version of something we can still do. +NICE = "nice" + +#: Neither. Recorded because the number is worth knowing and somebody will ask. +INFO = "info" + + +@dataclass(frozen=True) +class Check: + """One question, the code that answers it, and what a failure would cost.""" + + key: str + question: str + weight: str + costs: str + source: str + #: Set when we already know this one fails in the browser and have decided what to do + #: instead. The sentence goes in the report and the check stops failing the build, so a + #: known gap stays visible without hiding the next one behind a permanently red job. + accepted: str = "" + + @property + def blocking(self) -> bool: + """Should the build stop when this works natively and not in the browser.""" + return self.weight == TIER0 and not self.accepted + + +CHECKS = [ + Check( + key="version", + question="Which CPython is this, and what was it built for", + weight=INFO, + costs="Nothing. Every other answer here is about this interpreter.", + source=""" +import platform +import sys +import sysconfig + +result = { + "python": platform.python_version(), + "platform": sysconfig.get_platform(), + "pointer_bytes": sys.maxsize.bit_length() // 8 + 1, + "free_threaded": bool(sysconfig.get_config_var("Py_GIL_DISABLED")), +} +""", + ), + Check( + key="internal_capi_import", + question="Is _testinternalcapi importable at all", + weight=TIER0, + costs="Everything that shows the compiler one stage at a time. It is the single " + "best hook this project has, and without it T05 has nothing live to show.", + # The first two are called for real by the checks below. The third is only looked + # at, never called, because assemble_code_object asserts on its metadata instead of + # raising and a failed assertion aborts the process rather than throwing something + # a notebook could survive. Same reason pyxray.compiler.assemble does not call it. + source=""" +import _testinternalcapi + +wanted = ("compiler_codegen", "optimize_cfg", "assemble_code_object") +result = {name: hasattr(_testinternalcapi, name) for name in wanted} +""", + ), + Check( + key="compiler_codegen", + question="Does compiler_codegen turn a tree into an instruction sequence", + weight=TIER0, + costs="The first compiler stage in T05 and in the pipeline explorer widget.", + source=""" +import _testinternalcapi +import ast + +sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse("answer = 6 * 7"), "", 0) +instructions = sequence.get_instructions() +result = { + "instructions": len(instructions), + "first": instructions[0][0], + "metadata_keys": sorted(metadata), +} +""", + ), + Check( + key="optimize_cfg", + question="Does optimize_cfg run over that sequence the way pyxray calls it", + weight=TIER0, + costs="The middle stage. It is the one that makes 6 * 7 disappear, which is the " + "single most convincing thing in the first part of the course.", + accepted="The metadata this build hands back has no consts key, so the call fails " + "before it reaches the optimizer. The check below shows the optimizer itself is " + "fine, so in the browser the pipeline builds that list from the instruction " + "sequence instead of asking for it.", + source=""" +import _testinternalcapi +import ast + +sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse("answer = 6 * 7"), "", 0) +optimized = _testinternalcapi.optimize_cfg(sequence, metadata["consts"], 0) +result = {"instructions": len(optimized.get_instructions())} +""", + ), + Check( + key="optimize_cfg_direct", + # The check above asks for the constants in the metadata that codegen returned, + # which is what our own code does. If that key is missing the check fails without + # ever reaching the function. This one builds its own list of the right length off + # the instruction sequence, so the answer is about optimize_cfg and nothing else. + question="Does optimize_cfg run at all, given a constants list built by hand", + weight=TIER0, + costs="The same middle stage, asked in a way a missing metadata key cannot hide.", + source=""" +import _testinternalcapi +import ast +import opcode + +sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse("answer = 6 * 7"), "", 0) +load = opcode.opmap["LOAD_CONST"] +slots = [one[1] for one in sequence.get_instructions() if one[0] == load] +consts = [None] * (max(slots) + 1 if slots else 0) +optimized = _testinternalcapi.optimize_cfg(sequence, consts, 0) +result = {"slots": len(consts), "instructions": len(optimized.get_instructions())} +""", + ), + Check( + key="optimize_cfg_short_consts", + # Not a capability. A safety question, and the reason it is here is that a reader + # types their own code into the pipeline widget, and a wrong constants list is one + # of the easy ways to get there. + question="What happens when optimize_cfg is handed a constants list that is too short", + weight=INFO, + costs="Nothing, and it is worth knowing. A clean exception means the widget can " + "show the reader their mistake. A dead runtime means it has to check first.", + source=""" +import _testinternalcapi +import ast + +sequence, metadata = _testinternalcapi.compiler_codegen(ast.parse("answer = 6 * 7"), "", 0) +try: + _testinternalcapi.optimize_cfg(sequence, [], 0) + result = {"raised": None} +except BaseException as error: + result = {"raised": f"{type(error).__name__}: {error}"} +""", + ), + Check( + key="ctypes_header", + question="Can ctypes read the two fields in front of every object", + weight=TIER0, + costs="Object headers shown from a recording rather than from the reader's own " + "live objects. A real loss and survivable.", + source=""" +import ctypes +import sys + +value = [1, 2, 3] +word = ctypes.sizeof(ctypes.c_ssize_t) +result = { + "refcount_field": ctypes.c_ssize_t.from_address(id(value)).value, + "getrefcount": sys.getrefcount(value), + "type_pointer_matches": ctypes.c_void_p.from_address(id(value) + word).value == id(list), + "word_bytes": word, +} +""", + ), + Check( + key="monitoring", + question="Does sys.monitoring register a callback and fire it", + weight=TIER0, + costs="The stepper goes offline for Tier 0, and watching a function run one " + "instruction at a time is most of T07.", + source=""" +import sys + +TOOL = 5 +seen = [] +sys.monitoring.use_tool_id(TOOL, "wasmprobe") +try: + def target(): + return 1 + 1 + sys.monitoring.register_callback( + TOOL, sys.monitoring.events.PY_START, lambda *arguments: seen.append(arguments) + ) + sys.monitoring.set_local_events(TOOL, target.__code__, sys.monitoring.events.PY_START) + target() +finally: + sys.monitoring.free_tool_id(TOOL) +allowed = [ + name + for name in dir(sys.monitoring.events) + if name.isupper() and not name.startswith("NO_") +] +result = {"fired": len(seen), "event_names": len(allowed)} +""", + ), + Check( + key="settrace", + question="Does sys.settrace still see call, line and return", + weight=NICE, + costs="The fallback for anything sys.monitoring cannot do. Losing both would be " + "the bad case.", + source=""" +import sys + +seen = [] +def tracer(frame, event, argument): + seen.append(event) + return tracer +def target(): + return 2 +sys.settrace(tracer) +try: + target() +finally: + sys.settrace(None) +result = {"events": seen} +""", + ), + Check( + key="gc", + question="Does the cycle collector behave the way T09 says it does", + weight=TIER0, + costs="T09 shows a cycle being collected. If the numbers are different in the " + "browser the lesson is teaching Emscripten rather than CPython.", + source=""" +import gc +import weakref + +class Node: + pass +first, second = Node(), Node() +watch = weakref.ref(first) +first.other, second.other = second, first +del first, second +gc.collect() +# What gc.collect() returns counts everything it swept, including whatever else this +# process happened to be holding, so it is different every time. Whether this particular +# cycle went away is the question the lesson actually asks. +result = { + "cycle_freed": watch() is None, + "thresholds": list(gc.get_threshold()), + "enabled": gc.isenabled(), + "generations": len(gc.get_stats()), +} +""", + ), + Check( + key="debugmallocstats", + question="Does sys._debugmallocstats produce anything under Emscripten's allocator", + weight=NICE, + costs="One cell in T09. It writes to the real standard error rather than to " + "sys.stderr, so a notebook cannot capture it anyway.", + source=""" +import sys + +result = {"callable": callable(getattr(sys, "_debugmallocstats", None))} +""", + ), + Check( + key="front_end_modules", + question="Do dis, ast, symtable, tokenize and marshal all import", + weight=TIER0, + costs="Most of the first part of the course. These are pure Python or small C " + "modules, so a failure here would be a surprise.", + source=""" +found = {} +for name in ("dis", "ast", "symtable", "tokenize", "marshal", "opcode", "_opcode"): + try: + __import__(name) + found[name] = True + except Exception as error: + found[name] = f"{type(error).__name__}: {error}" +result = found +""", + ), + Check( + key="disassembly", + question="Does dis give the same instructions as a native interpreter", + weight=TIER0, + costs="Every bytecode listing in the course. A difference here is a difference in " + "the compiled build rather than in the language.", + source=""" +import dis +import marshal + +source = "answer = 6 * 7" +code = compile(source, "", "exec") +result = { + "opnames": [one.opname for one in dis.get_instructions(source)], + "code_size": len(code.co_code), + "consts": [repr(one) for one in code.co_consts], + "marshal_size": len(marshal.dumps(code)), +} +""", + ), + Check( + key="small_integers", + question="Where does the shared range of small integers stop", + weight=INFO, + costs="Nothing, and T08 measures it rather than asserting it for this reason.", + source=""" +top = 0 +for candidate in range(0, 4096): + if int(str(candidate)) is int(str(candidate)): + top = candidate +result = {"top": top} +""", + ), + Check( + key="threading", + question="Can a thread be started", + weight=NICE, + costs="The whole concurrency part in the browser. Those lessons are late enough " + "that a recording is an acceptable answer, and it has to be a deliberate one.", + source=""" +import threading + +ran = [] +worker = threading.Thread(target=lambda: ran.append(True)) +worker.start() +worker.join() +result = {"ran": ran == [True], "active": threading.active_count()} +""", + ), +] + +BY_KEY = {check.key: check for check in CHECKS} diff --git a/tools/wasmprobe/src/wasmprobe/cli.py b/tools/wasmprobe/src/wasmprobe/cli.py new file mode 100644 index 0000000..2201be3 --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/cli.py @@ -0,0 +1,159 @@ +"""The command line. + +Five subcommands, and they are meant to be run in this order. + + wasmprobe native --into probes/pyodide + wasmprobe browser --into probes/pyodide + wasmprobe report probes/pyodide --into probes/pyodide/report.md + wasmprobe notebook --into probes/pyodide/probe.ipynb + wasmprobe check probes/pyodide + +The last one is the one CI runs. It reads the two recordings and fails when a check the +lessons depend on works natively and not in the browser, which is the only difference worth +stopping a build over, and when the report or the notebook has fallen behind the checks. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from . import browser, native, notebook, report +from .checks import CHECKS +from .result import Run + +#: The file names inside the results directory. +NATIVE = "native.json" +BROWSER = "pyodide.json" +REPORT = "report.md" +NOTEBOOK = "probe.ipynb" + + +def _load(room: Path) -> tuple[Run, Run]: + here, there = room / NATIVE, room / BROWSER + for path in (here, there): + if not path.exists(): + raise SystemExit(f"no recording at {path}, run wasmprobe native and browser first") + return Run.load(here), Run.load(there) + + +def _native(args: argparse.Namespace) -> int: + run = native.run() + path = run.write(Path(args.into) / NATIVE) + worked = sum(1 for one in run.outcomes.values() if one.worked) + print(f"{path}: {worked} of {len(run.outcomes)} checks worked on CPython {run.python}") + return 0 + + +def _browser(args: argparse.Namespace) -> int: + problem = browser.ready() + if problem: + print(f"cannot run the browser probe: {problem}", file=sys.stderr) + return 1 + run = browser.run() + path = run.write(Path(args.into) / BROWSER) + worked = sum(1 for one in run.outcomes.values() if one.worked) + print(f"{path}: {worked} of {len(run.outcomes)} checks worked on Pyodide {run.python}") + return 0 + + +def _notebook(args: argparse.Namespace) -> int: + path = notebook.write(Path(args.into)) + print(f"{path}: the checks as a notebook somebody can run in their own browser") + return 0 + + +def _report(args: argparse.Namespace) -> int: + here, there = _load(Path(args.results)) + body = report.markdown(here, there) + if args.into: + path = Path(args.into) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + print(f"{path}: {report.summary(here, there)}") + else: + print(body, end="") + return 0 + + +def _stale(room: Path) -> list[str]: + """Generated files in the results directory that no longer match the checks.""" + here, there = _load(room) + wanted = {REPORT: report.markdown(here, there), NOTEBOOK: notebook.render()} + found = [] + for name, body in wanted.items(): + path = room / name + if not path.exists() or path.read_text(encoding="utf-8") != body: + found.append(name) + return found + + +def _check(args: argparse.Namespace) -> int: + room = Path(args.results) + here, there = _load(room) + print(report.summary(here, there)) + stale = _stale(room) + if stale: + print("", file=sys.stderr) + print(f"out of date: {', '.join(stale)}. Run just build-probe.", file=sys.stderr) + broken = report.regressions(here, there) + if not broken: + return 1 if stale else 0 + print("", file=sys.stderr) + print("Checks the lessons need that the browser cannot do:", file=sys.stderr) + for check in broken: + outcome = there.outcomes[check.key] + print(f" {check.key}: {outcome.status}. {check.costs}", file=sys.stderr) + print("", file=sys.stderr) + print( + "Record a fresh pair and update the report, or move those experiments to Tier 1.", + file=sys.stderr, + ) + return 1 + + +def _list(args: argparse.Namespace) -> int: + for check in CHECKS: + print(f"{check.key:22} {check.weight:6} {check.question}") + return 0 + + +def build() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="wasmprobe", description=__doc__) + subs = parser.add_subparsers(dest="command", required=True) + + here = subs.add_parser("native", help="run the checks on this interpreter") + here.add_argument("--into", default="probes/pyodide", help="directory to write the run into") + here.set_defaults(handler=_native) + + there = subs.add_parser("browser", help="run the checks inside Pyodide, through Node") + there.add_argument("--into", default="probes/pyodide", help="directory to write the run into") + there.set_defaults(handler=_browser) + + book = subs.add_parser("notebook", help="write the checks out as a runnable notebook") + book.add_argument("--into", default="probes/pyodide/probe.ipynb", help="file to write") + book.set_defaults(handler=_notebook) + + shown = subs.add_parser("report", help="render the matrix from two recordings") + shown.add_argument("results", nargs="?", default="probes/pyodide") + shown.add_argument("--into", default="", help="file to write, otherwise standard output") + shown.set_defaults(handler=_report) + + gate = subs.add_parser("check", help="fail when a Tier 0 check stopped working in the browser") + gate.add_argument("results", nargs="?", default="probes/pyodide") + gate.set_defaults(handler=_check) + + named = subs.add_parser("list", help="show every check and what it is for") + named.set_defaults(handler=_list) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build().parse_args(argv) + return int(args.handler(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/wasmprobe/src/wasmprobe/native.py b/tools/wasmprobe/src/wasmprobe/native.py new file mode 100644 index 0000000..9f0b2af --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/native.py @@ -0,0 +1,40 @@ +"""Running the checks on the interpreter this process is. + +The native run is the control. Without it a browser answer has nothing to be surprising +against, and half the interesting findings are of the form "this works everywhere except +there" rather than "this does not work". + +Each check gets its own namespace and is expected to leave its answer in `result`. That is +a small amount of ceremony in exchange for the checks being ordinary Python that somebody +can paste into a prompt to see what it does. +""" + +from __future__ import annotations + +import platform +import traceback + +from .checks import CHECKS, Check +from .result import OK, RAISED, Outcome, Run + + +def one(check: Check) -> Outcome: + """Run a single check, catching anything it throws.""" + namespace: dict = {} + try: + exec(check.source, namespace) + except BaseException: + # BaseException rather than Exception because a check that trips a recursion limit + # or gets interrupted is a result worth recording, not a crash of the probe. + line = traceback.format_exc().strip().splitlines()[-1] + return Outcome(check.key, RAISED, error=line) + return Outcome(check.key, OK, value=namespace.get("result")) + + +def run(checks: list[Check] | None = None) -> Run: + """Every check, in order, on this interpreter.""" + outcomes = {} + for check in checks if checks is not None else CHECKS: + outcomes[check.key] = one(check) + # No boot time to report. This interpreter was already running. + return Run(runtime="native", python=platform.python_version(), outcomes=outcomes) diff --git a/tools/wasmprobe/src/wasmprobe/notebook.py b/tools/wasmprobe/src/wasmprobe/notebook.py new file mode 100644 index 0000000..f828749 --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/notebook.py @@ -0,0 +1,172 @@ +"""The probe as a notebook, so anybody can run it in their own browser. + +The recordings in this directory were made by the Node driver, which is convenient and also +one step removed from what a reader will actually use. This notebook closes that gap: open +it in Colab, in JupyterLite, or in a local Jupyter, and it runs the same checks in whatever +Python is underneath and prints the same matrix. + +It carries the checks inside itself rather than importing them, because a reader in a +browser tab has not installed this project and should not have to. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .checks import CHECKS, Check + +#: Where the notebook lives, next to the recordings it can be compared against. +DESTINATION = Path("probes/pyodide/probe.ipynb") + +#: The repository, for the badge at the top. +REPOSITORY = "tamnd/cpython-internals" + +INTRO = """# Which parts of CPython work in a browser + +This project promises that the first experiments in every lesson run in a browser tab with nothing installed. That rests on Pyodide, which is CPython compiled to WebAssembly, and on the introspection surfaces the lessons poke at surviving that build. Some of them do not, and the point of this notebook is to find out which, on the exact runtime you are sitting in front of rather than on one we tested a while ago. + +Run every cell. It takes a few seconds and installs nothing. At the end you get a table of what worked here, and you can compare it against the recordings committed next to this file. + +One warning worth reading first. One of the checks below asks what happens when the bytecode optimizer is handed a constants list that is too short. On a normal build that raises a tidy exception. In a WebAssembly build it reads past the end of memory and takes the whole runtime with it, which in a notebook means the kernel dies and you have to restart it. That check is last for exactly this reason, and everything above it will have already printed. +""" + +CHECKS_INTRO = """## The checks + +Each one is a small piece of Python that leaves its answer in a variable called `result`. They are written out in full rather than imported, so you can read what is being asked, and edit one to ask something else. +""" + +RUN_INTRO = """## Running them + +Each check gets a fresh namespace and anything it throws is caught and recorded, so one failure does not stop the rest. A check that takes the runtime down cannot be caught, which is why the dangerous one is separated out below. +""" + +TABLE = ''' +def matrix(answers): + """Print what happened, one row per check.""" + width = max(len(key) for key in answers) + print(f"{'check'.ljust(width)} status answer") + print("-" * (width + 40)) + for key, (status, answer) in answers.items(): + print(f"{key.ljust(width)} {status.ljust(6)} {answer}") + + +matrix(answers) +''' + +RUNNER = """ +answers = {} +for key, source in CHECKS.items(): + namespace = {} + try: + exec(source, namespace) + except BaseException as error: + answers[key] = ("raised", f"{type(error).__name__}: {error}") + else: + answers[key] = ("ok", namespace.get("result")) + +print(f"{sum(1 for status, _ in answers.values() if status == 'ok')} of {len(answers)} worked") +""" + +DANGER = """## The one that can kill the kernel + +Everything above has already printed, so run this last. The cell catches the exception itself, so on a normal build you get a tidy `ValueError` naming the constant it could not find. In a browser there is nothing to catch: the read goes past the end of memory, the runtime does not come back, and you restart the kernel. + +That difference is the reason the pipeline widget in the lessons builds its own constants list rather than trusting the one it is handed. +""" + +TABLE_INTRO = """### The table + +The same shape as the matrix in `report.md` next to this notebook, so the two are easy to read against each other. The answer column is whatever the check left in `result`. +""" + +AFTERWARDS = """If the cell above came back rather than killing the kernel, here is what it caught. If the kernel died, that is the answer, and it is the one worth telling us about. +""" + +CLOSING = """## What to do with this + +Compare what you got against `report.md` in this directory, which is the same matrix recorded on a native CPython and on Pyodide under Node. + +A check that failed here and passed there is usually your environment: an old Pyodide, a sandbox that blocks threads, a Python built without the test modules. A check that failed in both is our problem, and worth an issue. +""" + + +def _cell(kind: str, source: str, key: str) -> dict: + body = { + "cell_type": kind, + "id": key, + "metadata": {}, + "source": source.strip("\n").splitlines(keepends=True), + } + if kind == "code": + body["execution_count"] = None + body["outputs"] = [] + # Keys in alphabetical order, the same reason the lesson builder does it: that is what + # nbformat writes, so opening this in Jupyter and saving does not reorder the file. + return dict(sorted(body.items())) + + +def _badge() -> str: + path = f"https://colab.research.google.com/github/{REPOSITORY}/blob/main/{DESTINATION}" + return f"[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)]({path})" + + +def _literal(checks: list[Check]) -> str: + """The checks as a dictionary somebody can read and edit, one entry per check.""" + lines = ["CHECKS = {"] + for check in checks: + lines.append(f" # {check.question}") + lines.append(f' "{check.key}": """{check.source}""",') + lines.append("}") + lines.append("") + lines.append('print(f"{len(CHECKS)} checks")') + return "\n".join(lines) + + +def build(checks: list[Check] | None = None) -> dict: + """The notebook, as the dictionary that gets written out as JSON.""" + wanted = checks if checks is not None else CHECKS + # The dangerous one runs on its own at the end, after everything else has printed. + safe = [check for check in wanted if check.key != "optimize_cfg_short_consts"] + risky = [check for check in wanted if check.key == "optimize_cfg_short_consts"] + + written = [ + ("markdown", INTRO + "\n" + _badge()), + ("markdown", CHECKS_INTRO), + ("code", _literal(safe)), + ("markdown", RUN_INTRO), + ("code", RUNNER), + ("markdown", TABLE_INTRO), + ("code", TABLE), + ] + if risky: + written.append(("markdown", DANGER)) + written.append(("code", risky[0].source)) + written.append(("markdown", AFTERWARDS)) + written.append(("code", "print(result)")) + written.append(("markdown", CLOSING)) + cells = [ + _cell(kind, source, f"probe-{number:02d}") + for number, (kind, source) in enumerate(written, start=1) + ] + + return { + "cells": cells, + "metadata": { + "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, + "language_info": {"name": "python"}, + }, + "nbformat": 4, + "nbformat_minor": 5, + } + + +def render(checks: list[Check] | None = None) -> str: + return json.dumps(build(checks), indent=1, ensure_ascii=False) + "\n" + + +def write(path: Path | None = None, checks: list[Check] | None = None) -> Path: + destination = path or DESTINATION + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(render(checks), encoding="utf-8") + return destination diff --git a/tools/wasmprobe/src/wasmprobe/report.py b/tools/wasmprobe/src/wasmprobe/report.py new file mode 100644 index 0000000..55f382c --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/report.py @@ -0,0 +1,157 @@ +"""Turning two runs into a table somebody can read. + +The interesting column is not "did it work in the browser" on its own. It is the pair. A +check that fails in both places is a check that is wrong, or a surface that went away in +this version of Python, and either way it is our problem rather than WebAssembly's. A check +that works natively and fails in the browser is the actual finding. + +The output is markdown because it gets committed next to the results and read on GitHub. +""" + +from __future__ import annotations + +from .checks import CHECKS, Check +from .result import FATAL, OK, RAISED, SKIPPED, Outcome, Run + +#: What the pair of statuses means, in the order the report should worry about them. +BOTH_FINE = "works in both" +ONLY_NATIVE = "works natively, not in the browser" +ONLY_BROWSER = "works in the browser, not natively" +NEITHER = "fails in both" + +MARKS = {OK: "yes", RAISED: "raises", FATAL: "kills the runtime", SKIPPED: "not reached"} + + +def verdict(native: Outcome, browser: Outcome) -> str: + """One sentence about what the two runs together say.""" + if native.worked and browser.worked: + return BOTH_FINE + if native.worked: + return ONLY_NATIVE + if browser.worked: + return ONLY_BROWSER + return NEITHER + + +def regressions(native: Run, browser: Run, checks: list[Check] | None = None) -> list[Check]: + """The blocking checks that work here and not there, which is what CI should fail on.""" + found = [] + for check in checks if checks is not None else CHECKS: + if not check.blocking: + continue + here = native.outcomes.get(check.key) + there = browser.outcomes.get(check.key) + if here is None or there is None: + continue + if verdict(here, there) == ONLY_NATIVE: + found.append(check) + return found + + +def differences( + native: Run, browser: Run, checks: list[Check] | None = None +) -> list[tuple[Check, object, object]]: + """Checks that worked in both places and came back with different answers. + + These are not failures. They are the places where the two builds really are different + machines, and a lesson that asserts one of these numbers is teaching the build rather + than the language. + """ + found = [] + for check in checks if checks is not None else CHECKS: + here = native.outcomes.get(check.key) + there = browser.outcomes.get(check.key) + if here is None or there is None or not (here.worked and there.worked): + continue + if here.value != there.value: + found.append((check, here.value, there.value)) + return found + + +def _cell(outcome: Outcome | None) -> str: + if outcome is None: + return "not run" + mark = MARKS.get(outcome.status, outcome.status) + if outcome.worked or not outcome.error: + return mark + return f"{mark}, {outcome.error}" + + +def table(native: Run, browser: Run, checks: list[Check] | None = None) -> str: + """The matrix, one row per check.""" + here_column = f"{native.python} native" + there_column = f"{browser.python} WebAssembly" + rows = [ + f"| Check | Needed for | {here_column} | {there_column} |", + "| --- | --- | --- | --- |", + ] + for check in checks if checks is not None else CHECKS: + here = _cell(native.outcomes.get(check.key)) + there = _cell(browser.outcomes.get(check.key)) + needed = check.weight + (", known gap" if check.accepted else "") + rows.append(f"| {check.question} | {needed} | {here} | {there} |") + return "\n".join(rows) + + +def summary(native: Run, browser: Run, checks: list[Check] | None = None) -> str: + """A line of counts, so the table has something to be read against.""" + wanted = checks if checks is not None else CHECKS + counts = {BOTH_FINE: 0, ONLY_NATIVE: 0, ONLY_BROWSER: 0, NEITHER: 0} + for check in wanted: + here = native.outcomes.get(check.key) + there = browser.outcomes.get(check.key) + if here is None or there is None: + continue + counts[verdict(here, there)] += 1 + parts = [f"{count} {name}" for name, count in counts.items() if count] + blocked = len(regressions(native, browser, wanted)) + tail = f", {blocked} of them blocking" if blocked else "" + return f"{len(wanted)} checks: " + ", ".join(parts) + tail + "." + + +def markdown(native: Run, browser: Run, checks: list[Check] | None = None) -> str: + """The whole report, ready to be written to a file.""" + # Written as one output line, in pieces, because the project keeps prose to one + # sentence per line and the line length rule still applies to the source. + megabytes = browser.payload_bytes / 1_000_000 + intro = ( + f"Generated by `wasmprobe report`. " + f"Native run on CPython {native.python}, " + f"WebAssembly run on Pyodide with CPython {browser.python}, " + f"which booted in {browser.seconds:.1f} seconds " + f"off a local disk after {megabytes:.1f} MB of runtime and standard library." + ) + lines = [ + "# What works under Pyodide", + "", + intro, + "", + summary(native, browser, checks), + "", + table(native, browser, checks), + "", + "## What a failure would cost", + "", + ] + for check in checks if checks is not None else CHECKS: + lines.append(f"**{check.question}.** {check.costs}") + lines.append("") + apart = differences(native, browser, checks) + if apart: + lines.append("## Where the two answers differ") + lines.append("") + lines.append("Both runtimes answered these, and answered them differently.") + lines.append("") + lines.append("| Check | Native | WebAssembly |") + lines.append("| --- | --- | --- |") + for check, here, there in apart: + lines.append(f"| {check.question} | `{here}` | `{there}` |") + lines.append("") + known = [check for check in (checks if checks is not None else CHECKS) if check.accepted] + if known: + lines.append("## Known gaps, and what we do instead") + lines.append("") + for check in known: + lines.append(f"**{check.question}.** {check.accepted}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" diff --git a/tools/wasmprobe/src/wasmprobe/result.py b/tools/wasmprobe/src/wasmprobe/result.py new file mode 100644 index 0000000..31427bb --- /dev/null +++ b/tools/wasmprobe/src/wasmprobe/result.py @@ -0,0 +1,101 @@ +"""What one run of the checks produced, and how to keep it on disk. + +A run is a plain dictionary of check key to outcome, plus a note saying which runtime made +it. Two runs are compared by the report, so both sides use this shape: the native one this +process produces and the WebAssembly one the Node driver hands back as JSON. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +#: The check ran and returned something. +OK = "ok" + +#: The check raised. The runtime is still standing and the next check can go ahead. +RAISED = "raised" + +#: The check took the whole runtime down with it. Nothing after it ran in that runtime, so +#: the driver starts a new one and carries on from the next check. +FATAL = "fatal" + +#: Never got as far as running, because the run stopped early. +SKIPPED = "skipped" + + +@dataclass(frozen=True) +class Outcome: + """One check's answer on one runtime.""" + + key: str + status: str + value: object = None + error: str = "" + + @property + def worked(self) -> bool: + return self.status == OK + + def as_dict(self) -> dict: + body = {"key": self.key, "status": self.status} + if self.value is not None: + body["value"] = self.value + if self.error: + body["error"] = self.error + return body + + @classmethod + def from_dict(cls, body: dict) -> Outcome: + return cls( + key=str(body["key"]), + status=str(body["status"]), + value=body.get("value"), + error=str(body.get("error", "")), + ) + + +@dataclass(frozen=True) +class Run: + """Every outcome from one runtime, and enough about it to name it in a report.""" + + runtime: str + python: str + outcomes: dict[str, Outcome] = field(default_factory=dict) + + #: How long the runtime took to become ready to run the first check. Zero for the + #: interpreter this process already is. + seconds: float = 0.0 + + #: What a browser has to download before any of this can start. Zero for a runtime that + #: was already on the machine. This is the number that decides whether somebody on a + #: phone waits or closes the tab, more than the boot time does. + payload_bytes: int = 0 + + def as_json(self) -> str: + body = { + "runtime": self.runtime, + "python": self.python, + "seconds": round(self.seconds, 3), + "payload_bytes": self.payload_bytes, + "outcomes": [one.as_dict() for one in self.outcomes.values()], + } + return json.dumps(body, indent=2, sort_keys=False) + "\n" + + @classmethod + def load(cls, path: Path) -> Run: + body = json.loads(path.read_text(encoding="utf-8")) + outcomes = {one["key"]: Outcome.from_dict(one) for one in body["outcomes"]} + return cls( + runtime=str(body["runtime"]), + python=str(body["python"]), + outcomes=outcomes, + seconds=float(body.get("seconds", 0.0)), + payload_bytes=int(body.get("payload_bytes", 0)), + ) + + def write(self, path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.as_json(), encoding="utf-8") + return path diff --git a/tools/wasmprobe/tests/test_wasmprobe_browser.py b/tools/wasmprobe/tests/test_wasmprobe_browser.py new file mode 100644 index 0000000..4646e4d --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_browser.py @@ -0,0 +1,139 @@ +"""The Node side, and the committed recordings it produced. + +The slow test here really does boot a WebAssembly Python, so it only runs when Node and the +Pyodide package are both installed. Everything else works off the recordings in the +repository, which is the point of committing them. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from wasmprobe import browser, report +from wasmprobe.checks import BY_KEY, CHECKS +from wasmprobe.result import FATAL, OK, SKIPPED, Outcome, Run + +RESULTS = Path(__file__).resolve().parents[3] / "probes" / "pyodide" + + +def test_the_driver_is_next_to_the_package(): + assert browser.DRIVER.name == "driver.mjs" + assert browser.DRIVER.exists() + + +def test_a_missing_check_comes_back_as_skipped(): + body = {"runtime": "pyodide", "python": "3.14.2", "seconds": 1.0, "outcomes": []} + run = browser._assemble(body, [BY_KEY["version"]], 0) + assert run.outcomes["version"].status == SKIPPED + + +def test_the_order_of_the_checks_is_the_order_of_the_report(): + keys = ["version", "gc"] + body = { + "runtime": "pyodide", + "python": "3.14.2", + "seconds": 1.0, + "outcomes": [{"key": "gc", "status": OK}, {"key": "version", "status": OK}], + } + run = browser._assemble(body, [BY_KEY[key] for key in keys], 0) + assert list(run.outcomes) == keys + + +def test_a_driver_that_died_gets_a_better_sentence(): + body = { + "runtime": "pyodide", + "python": "3.14.2", + "seconds": 1.0, + "outcomes": [{"key": "gc", "status": FATAL, "error": "the runtime did not come back"}], + } + run = browser._assemble(body, [BY_KEY["gc"]], 1) + assert run.outcomes["gc"].error == "took the whole process down" + + +def test_ready_says_what_is_missing(monkeypatch): + monkeypatch.setattr(browser.shutil, "which", lambda name: None) + assert browser.ready() == "node is not on PATH" + + +def test_running_without_node_raises_rather_than_pretending(monkeypatch): + monkeypatch.setattr(browser, "ready", lambda: "node is not on PATH") + with pytest.raises(browser.Missing): + browser.run() + + +def recorded(): + """The two files in probes/pyodide, which are the committed answer to the gate.""" + for name in ("native.json", "pyodide.json"): + if not (RESULTS / name).exists(): + pytest.skip(f"no recording at {RESULTS / name}") + return Run.load(RESULTS / "native.json"), Run.load(RESULTS / "pyodide.json") + + +def test_both_recordings_cover_every_check(): + native, browser_run = recorded() + keys = {check.key for check in CHECKS} + assert set(native.outcomes) == keys + assert set(browser_run.outcomes) == keys + + +def test_nothing_blocking_is_broken_in_the_recorded_browser_run(): + native, browser_run = recorded() + assert report.regressions(native, browser_run) == [] + + +def test_the_report_on_disk_matches_the_recordings(): + native, browser_run = recorded() + path = RESULTS / "report.md" + if not path.exists(): + pytest.skip("no report yet") + assert path.read_text(encoding="utf-8") == report.markdown(native, browser_run) + + +def test_the_recorded_browser_run_really_was_webassembly(): + _, browser_run = recorded() + assert browser_run.outcomes["version"].value["platform"].startswith("emscripten") + + +def test_the_native_recording_answered_everything(): + """It is the control. Anything broken there is our bug rather than a WebAssembly one.""" + native, _ = recorded() + broken = {key: one.error for key, one in native.outcomes.items() if not one.worked} + assert not broken + + +@pytest.mark.slow +def test_a_real_webassembly_run(): + problem = browser.ready() + if problem: + pytest.skip(problem) + run = browser.run([BY_KEY["version"], BY_KEY["front_end_modules"]]) + assert run.runtime == "pyodide" + assert run.seconds > 0 + assert run.outcomes["version"].value["platform"].startswith("emscripten") + assert run.outcomes["front_end_modules"].value["dis"] is True + + +@pytest.mark.slow +def test_a_check_that_kills_the_runtime_does_not_take_the_rest_with_it(): + """The whole reason the driver reboots between checks.""" + problem = browser.ready() + if problem: + pytest.skip(problem) + run = browser.run([BY_KEY["optimize_cfg_short_consts"], BY_KEY["front_end_modules"]]) + assert run.outcomes["optimize_cfg_short_consts"].status == FATAL + assert run.outcomes["front_end_modules"].status == OK + + +def test_the_recordings_are_indented_json_a_person_can_read(): + path = RESULTS / "pyodide.json" + if not path.exists(): + pytest.skip("no recording") + text = path.read_text(encoding="utf-8") + assert "\n " in text + assert json.loads(text)["runtime"] == "pyodide" + + +def test_outcome_equality_is_by_value(): + assert Outcome("a", OK, value=[1]) == Outcome("a", OK, value=[1]) diff --git a/tools/wasmprobe/tests/test_wasmprobe_checks.py b/tools/wasmprobe/tests/test_wasmprobe_checks.py new file mode 100644 index 0000000..11abf8d --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_checks.py @@ -0,0 +1,82 @@ +"""The checks have to be well formed, and they have to actually run somewhere.""" + +from __future__ import annotations + +import ast +import builtins + +import pytest +from wasmprobe.checks import BY_KEY, CHECKS, INFO, NICE, TIER0, Check + +WEIGHTS = {TIER0, NICE, INFO} + + +def test_keys_are_unique(): + keys = [check.key for check in CHECKS] + assert len(keys) == len(set(keys)) + assert set(BY_KEY) == set(keys) + + +@pytest.mark.parametrize("check", CHECKS, ids=lambda check: check.key) +def test_shape(check: Check): + assert check.weight in WEIGHTS + assert check.question and not check.question.endswith(".") + assert check.costs.endswith(".") + assert "result" in check.source + + +@pytest.mark.parametrize("check", CHECKS, ids=lambda check: check.key) +def test_prose_has_no_dashes(check: Check): + """The questions and the costs end up in a committed report, so the style rules apply.""" + for text in (check.question, check.costs, check.accepted): + assert "\u2014" not in text + assert "\u2013" not in text + + +def test_only_tier_zero_can_be_accepted(): + """Accepting a gap only means something for a check that would otherwise fail a build.""" + for check in CHECKS: + if check.accepted: + assert check.weight == TIER0 + + +def test_accepted_checks_do_not_block(): + accepted = [check for check in CHECKS if check.accepted] + assert accepted, "the point of the field is that at least one gap is known" + assert not any(check.blocking for check in accepted) + + +def test_tier_zero_without_an_excuse_blocks(): + plain = [check for check in CHECKS if check.weight == TIER0 and not check.accepted] + assert plain and all(check.blocking for check in plain) + + +@pytest.mark.parametrize("check", CHECKS, ids=lambda check: check.key) +def test_a_check_imports_what_it_uses(check: Check): + """Nothing carries over, because a check that crashes leaves the next one a new runtime.""" + tree = ast.parse(check.source) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import | ast.ImportFrom): + imported.update(alias.asname or alias.name for alias in node.names) + used = { + node.value.id + for node in ast.walk(tree) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) + } + assigned = { + node.id + for node in ast.walk(tree) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + } + assigned |= {node.name for node in ast.walk(tree) if isinstance(node, ast.ExceptHandler)} + assigned |= { + node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef | ast.FunctionDef) + } + known = imported | assigned | set(dir(builtins)) + assert used <= known, f"{check.key} reaches for {sorted(used - known)} without importing it" + + +def test_a_check_parses(): + for check in CHECKS: + ast.parse(check.source) diff --git a/tools/wasmprobe/tests/test_wasmprobe_cli.py b/tools/wasmprobe/tests/test_wasmprobe_cli.py new file mode 100644 index 0000000..b0936f1 --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_cli.py @@ -0,0 +1,111 @@ +"""The command line, driven the way CI drives it.""" + +from __future__ import annotations + +import pytest +from wasmprobe import notebook, report +from wasmprobe.cli import BROWSER, NATIVE, NOTEBOOK, REPORT, main +from wasmprobe.result import FATAL, OK, Outcome, Run + + +def pair(room, browser_status=OK): + native = Run("native", "3.15.0", {"gc": Outcome("gc", OK)}) + browser = Run( + "pyodide", + "3.14.2", + {"gc": Outcome("gc", browser_status, error="boom" if browser_status != OK else "")}, + seconds=1.4, + ) + native.write(room / NATIVE) + browser.write(room / BROWSER) + return native, browser + + +def generated(room, checks): + """Write the two generated files, so the staleness gate has nothing to complain about.""" + native, browser = Run.load(room / NATIVE), Run.load(room / BROWSER) + (room / REPORT).write_text(report.markdown(native, browser, checks), encoding="utf-8") + (room / NOTEBOOK).write_text(notebook.render(), encoding="utf-8") + + +def only_gc(monkeypatch): + """Trim the check list the report walks, so the fixtures above are the whole world.""" + from wasmprobe.checks import BY_KEY + + wanted = [BY_KEY["gc"]] + monkeypatch.setattr(report, "CHECKS", wanted) + return wanted + + +def test_list_prints_every_check(capsys): + assert main(["list"]) == 0 + printed = capsys.readouterr().out + assert "internal_capi_import" in printed + assert printed.count("\n") >= 10 + + +def test_native_writes_a_run(tmp_path, capsys): + assert main(["native", "--into", str(tmp_path)]) == 0 + assert (tmp_path / NATIVE).exists() + assert "checks worked on CPython" in capsys.readouterr().out + + +def test_report_without_a_destination_prints(tmp_path, capsys, monkeypatch): + only_gc(monkeypatch) + pair(tmp_path) + assert main(["report", str(tmp_path)]) == 0 + assert "# What works under Pyodide" in capsys.readouterr().out + + +def test_report_into_a_file(tmp_path, monkeypatch): + only_gc(monkeypatch) + pair(tmp_path) + destination = tmp_path / "deep" / "report.md" + assert main(["report", str(tmp_path), "--into", str(destination)]) == 0 + assert "3.14.2 WebAssembly" in destination.read_text(encoding="utf-8") + + +def test_check_passes_when_nothing_regressed(tmp_path, monkeypatch, capsys): + wanted = only_gc(monkeypatch) + pair(tmp_path) + generated(tmp_path, wanted) + assert main(["check", str(tmp_path)]) == 0 + assert "works in both" in capsys.readouterr().out + + +def test_check_fails_when_the_report_has_fallen_behind(tmp_path, monkeypatch, capsys): + only_gc(monkeypatch) + pair(tmp_path) + assert main(["check", str(tmp_path)]) == 1 + assert "out of date: report.md, probe.ipynb" in capsys.readouterr().err + + +def test_notebook_writes_where_it_is_told(tmp_path, capsys): + destination = tmp_path / "probe.ipynb" + assert main(["notebook", "--into", str(destination)]) == 0 + assert destination.exists() + assert "run in their own browser" in capsys.readouterr().out + + +def test_check_fails_and_says_what_it_costs(tmp_path, monkeypatch, capsys): + wanted = only_gc(monkeypatch) + pair(tmp_path, browser_status=FATAL) + generated(tmp_path, wanted) + assert main(["check", str(tmp_path)]) == 1 + complaint = capsys.readouterr().err + assert "gc: fatal" in complaint + assert "move those experiments to Tier 1" in complaint + + +def test_a_missing_recording_says_which_one(tmp_path): + with pytest.raises(SystemExit) as stopped: + main(["check", str(tmp_path)]) + assert "native.json" in str(stopped.value) + + +def test_browser_without_node_is_a_failure_not_a_crash(capsys, monkeypatch): + from wasmprobe import browser + + monkeypatch.setattr(browser, "ready", lambda: "node is not on PATH") + assert main(["browser", "--into", "unused"]) == 1 + assert "node is not on PATH" in capsys.readouterr().err diff --git a/tools/wasmprobe/tests/test_wasmprobe_native.py b/tools/wasmprobe/tests/test_wasmprobe_native.py new file mode 100644 index 0000000..7c95842 --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_native.py @@ -0,0 +1,66 @@ +"""Running the checks here, which is also the control the browser run is compared against.""" + +from __future__ import annotations + +import pytest +from wasmprobe.checks import CHECKS, INFO, Check +from wasmprobe.native import one, run +from wasmprobe.result import OK, RAISED + +BOOM = Check( + key="boom", + question="What does a check that throws look like", + weight=INFO, + costs="Nothing. It is here for the tests.", + source="raise ValueError('no')", +) + +QUIET = Check( + key="quiet", + question="What does a check that answers nothing look like", + weight=INFO, + costs="Nothing. It is here for the tests.", + source="pass", +) + + +def test_a_check_that_works(): + outcome = one(CHECKS[0]) + assert outcome.status == OK + assert outcome.worked + assert outcome.value["python"] + + +def test_a_check_that_throws_is_recorded_rather_than_raised(): + outcome = one(BOOM) + assert outcome.status == RAISED + assert not outcome.worked + assert outcome.error == "ValueError: no" + + +def test_a_check_that_never_sets_result(): + outcome = one(QUIET) + assert outcome.status == OK + assert outcome.value is None + + +def test_every_check_works_on_this_interpreter(): + """The native side is the control, so a failure here is our bug rather than a finding.""" + finished = run() + broken = {key: value.error for key, value in finished.outcomes.items() if not value.worked} + assert not broken + + +@pytest.mark.parametrize("check", CHECKS, ids=lambda check: check.key) +def test_answers_survive_json(check: Check): + """Whatever a check returns has to fit through the pipe from the Node driver.""" + import json + + json.dumps(one(check).value) + + +def test_run_reports_this_interpreter(): + finished = run([CHECKS[0]]) + assert finished.runtime == "native" + assert finished.python.count(".") == 2 + assert list(finished.outcomes) == [CHECKS[0].key] diff --git a/tools/wasmprobe/tests/test_wasmprobe_notebook.py b/tools/wasmprobe/tests/test_wasmprobe_notebook.py new file mode 100644 index 0000000..77f7dd1 --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_notebook.py @@ -0,0 +1,102 @@ +"""The notebook is generated and committed, so something has to check it did not drift.""" + +from __future__ import annotations + +import ast +import itertools +import json +from pathlib import Path + +import pytest +from wasmprobe import notebook +from wasmprobe.checks import CHECKS + +COMMITTED = Path(__file__).resolve().parents[3] / notebook.DESTINATION + + +def sources(kind: str) -> list[str]: + return [ + "".join(cell["source"]) for cell in notebook.build()["cells"] if cell["cell_type"] == kind + ] + + +def test_it_is_a_notebook_jupyter_will_open(): + body = notebook.build() + assert body["nbformat"] == 4 + assert body["metadata"]["kernelspec"]["name"] == "python3" + assert all(cell["id"] for cell in body["cells"]) + + +def test_cell_ids_are_unique_and_in_order(): + ids = [cell["id"] for cell in notebook.build()["cells"]] + assert ids == sorted(ids) + assert len(ids) == len(set(ids)) + + +def test_every_code_cell_parses(): + for source in sources("code"): + ast.parse(source) + + +def test_every_code_cell_has_a_markdown_cell_in_front_of_it(): + """The same rule the lessons follow. A reader should never meet a cell unannounced.""" + kinds = [cell["cell_type"] for cell in notebook.build()["cells"]] + assert kinds[0] == "markdown" + for before, after in itertools.pairwise(kinds): + if after == "code": + assert before in {"markdown", "code"} + + +def test_the_dangerous_check_is_last_and_on_its_own(): + """It kills the runtime in a browser, so everything else has to have printed first.""" + body = notebook.build() + keys = [cell["id"] for cell in body["cells"]] + holding = [ + index + for index, cell in enumerate(body["cells"]) + if "optimize_cfg(sequence, [], 0)" in "".join(cell["source"]) + ] + assert len(holding) == 1 + assert holding[0] >= len(keys) - 4 + + +def test_the_safe_checks_are_all_in_one_cell(): + body = "\n".join(sources("code")) + for check in CHECKS: + if check.key == "optimize_cfg_short_consts": + continue + assert f'"{check.key}"' in body + + +def test_it_installs_nothing(): + """A reader in a locked down browser tab has no package manager, and needs none.""" + body = "\n".join(sources("code")) + assert "pip install" not in body + assert "micropip" not in body + + +def test_the_prose_follows_the_house_style(): + for source in sources("markdown"): + assert "\u2014" not in source + assert "\u2013" not in source + + +def test_the_colab_badge_points_at_the_committed_path(): + first = "".join(notebook.build()["cells"][0]["source"]) + assert "colab.research.google.com/github/tamnd/cpython-internals" in first + assert str(notebook.DESTINATION) in first + + +def test_render_is_stable(): + assert notebook.render() == notebook.render() + + +def test_write_goes_where_it_is_told(tmp_path): + path = notebook.write(tmp_path / "deep" / "probe.ipynb") + assert json.loads(path.read_text(encoding="utf-8"))["nbformat"] == 4 + + +def test_the_committed_notebook_matches_the_checks(): + if not COMMITTED.exists(): + pytest.skip("no committed notebook") + assert COMMITTED.read_text(encoding="utf-8") == notebook.render() diff --git a/tools/wasmprobe/tests/test_wasmprobe_report.py b/tools/wasmprobe/tests/test_wasmprobe_report.py new file mode 100644 index 0000000..581fd4b --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_report.py @@ -0,0 +1,153 @@ +"""The table, the summary line, and the one thing that fails a build.""" + +from __future__ import annotations + +import pytest +from wasmprobe.checks import INFO, TIER0, Check +from wasmprobe.report import ( + BOTH_FINE, + NEITHER, + ONLY_BROWSER, + ONLY_NATIVE, + differences, + markdown, + regressions, + summary, + table, + verdict, +) +from wasmprobe.result import FATAL, OK, RAISED, Outcome, Run + + +def check(key, weight=TIER0, accepted=""): + return Check( + key=key, + question=f"Does {key} work", + weight=weight, + costs="Something.", + source="result = 1", + accepted=accepted, + ) + + +def runs(here, there): + native = Run("native", "3.15.0", {key: value for key, value in here.items()}) + browser = Run("pyodide", "3.14.2", {key: value for key, value in there.items()}, seconds=1.4) + return native, browser + + +@pytest.mark.parametrize( + ("here", "there", "expected"), + [ + (OK, OK, BOTH_FINE), + (OK, FATAL, ONLY_NATIVE), + (OK, RAISED, ONLY_NATIVE), + (RAISED, OK, ONLY_BROWSER), + (RAISED, RAISED, NEITHER), + ], +) +def test_verdict(here, there, expected): + assert verdict(Outcome("a", here), Outcome("a", there)) == expected + + +def test_a_blocking_check_that_broke_in_the_browser_is_a_regression(): + one = check("compiler") + native, browser = runs( + {"compiler": Outcome("compiler", OK)}, + {"compiler": Outcome("compiler", FATAL, error="boom")}, + ) + assert regressions(native, browser, [one]) == [one] + + +def test_an_accepted_gap_is_not_a_regression(): + one = check("optimizer", accepted="We build the list ourselves instead.") + native, browser = runs( + {"optimizer": Outcome("optimizer", OK)}, + {"optimizer": Outcome("optimizer", RAISED, error="KeyError")}, + ) + assert regressions(native, browser, [one]) == [] + + +def test_a_nice_to_have_is_not_a_regression(): + one = check("threads", weight=INFO) + native, browser = runs( + {"threads": Outcome("threads", OK)}, {"threads": Outcome("threads", RAISED, error="no")} + ) + assert regressions(native, browser, [one]) == [] + + +def test_broken_in_both_places_is_our_problem_not_a_regression(): + one = check("gone") + native, browser = runs( + {"gone": Outcome("gone", RAISED, error="no")}, {"gone": Outcome("gone", RAISED, error="no")} + ) + assert regressions(native, browser, [one]) == [] + + +def test_a_check_missing_from_one_side_is_skipped(): + one = check("half") + native, browser = runs({"half": Outcome("half", OK)}, {}) + assert regressions(native, browser, [one]) == [] + + +def test_differences_only_looks_at_checks_that_worked_in_both(): + same = check("same") + apart = check("apart") + broken = check("broken") + native, browser = runs( + { + "same": Outcome("same", OK, value=1), + "apart": Outcome("apart", OK, value=8), + "broken": Outcome("broken", OK, value=2), + }, + { + "same": Outcome("same", OK, value=1), + "apart": Outcome("apart", OK, value=4), + "broken": Outcome("broken", FATAL, error="boom"), + }, + ) + assert differences(native, browser, [same, apart, broken]) == [(apart, 8, 4)] + + +def test_summary_counts_and_names_the_blocking_ones(): + good = check("good") + bad = check("bad") + native, browser = runs( + {"good": Outcome("good", OK), "bad": Outcome("bad", OK)}, + {"good": Outcome("good", OK), "bad": Outcome("bad", FATAL, error="boom")}, + ) + line = summary(native, browser, [good, bad]) + assert line == f"2 checks: 1 {BOTH_FINE}, 1 {ONLY_NATIVE}, 1 of them blocking." + + +def test_the_table_has_a_row_for_every_check_and_names_both_pythons(): + one, two = check("one"), check("two") + native, browser = runs( + {"one": Outcome("one", OK), "two": Outcome("two", OK)}, + {"one": Outcome("one", OK), "two": Outcome("two", RAISED, error="KeyError: nope")}, + ) + body = table(native, browser, [one, two]) + lines = body.splitlines() + assert len(lines) == 4 + assert "3.15.0 native" in lines[0] + assert "3.14.2 WebAssembly" in lines[0] + assert "raises, KeyError: nope" in lines[3] + + +def test_the_report_mentions_the_boot_time_and_the_known_gaps(): + one = check("optimizer", accepted="We build the list ourselves instead.") + native, browser = runs( + {"optimizer": Outcome("optimizer", OK)}, + {"optimizer": Outcome("optimizer", RAISED, error="KeyError")}, + ) + body = markdown(native, browser, [one]) + assert "booted in 1.4 seconds" in body + assert "Known gaps" in body + assert "We build the list ourselves instead." in body + assert body.endswith("\n") + + +def test_no_known_gaps_means_no_section(): + one = check("fine") + native, browser = runs({"fine": Outcome("fine", OK)}, {"fine": Outcome("fine", OK)}) + assert "Known gaps" not in markdown(native, browser, [one]) diff --git a/tools/wasmprobe/tests/test_wasmprobe_result.py b/tools/wasmprobe/tests/test_wasmprobe_result.py new file mode 100644 index 0000000..e3db701 --- /dev/null +++ b/tools/wasmprobe/tests/test_wasmprobe_result.py @@ -0,0 +1,52 @@ +"""Outcomes and runs, and the round trip through disk that the two sides share.""" + +from __future__ import annotations + +import json + +from wasmprobe.result import FATAL, OK, RAISED, SKIPPED, Outcome, Run + + +def test_worked_is_only_true_for_ok(): + assert Outcome("a", OK).worked + for status in (RAISED, FATAL, SKIPPED): + assert not Outcome("a", status, error="why").worked + + +def test_empty_fields_stay_out_of_the_file(): + body = Outcome("a", OK).as_dict() + assert body == {"key": "a", "status": OK} + + +def test_a_value_and_an_error_both_survive(): + body = Outcome("a", RAISED, value=[1], error="ValueError: no").as_dict() + assert Outcome.from_dict(body) == Outcome("a", RAISED, value=[1], error="ValueError: no") + + +def test_a_run_round_trips(tmp_path): + run = Run( + runtime="pyodide", + python="3.14.2", + outcomes={ + "first": Outcome("first", OK, value={"n": 1}), + "second": Outcome("second", FATAL, error="memory access out of bounds"), + }, + seconds=1.25, + ) + path = run.write(tmp_path / "deep" / "run.json") + assert path.exists() + assert Run.load(path) == run + + +def test_the_file_is_readable_json_with_a_trailing_newline(tmp_path): + run = Run(runtime="native", python="3.15.0", outcomes={"a": Outcome("a", OK)}) + path = run.write(tmp_path / "run.json") + text = path.read_text(encoding="utf-8") + assert text.endswith("\n") + assert json.loads(text)["outcomes"] == [{"key": "a", "status": OK}] + + +def test_order_is_kept(tmp_path): + keys = ["c", "a", "b"] + run = Run("native", "3.15.0", {key: Outcome(key, OK) for key in keys}) + assert list(Run.load(run.write(tmp_path / "run.json")).outcomes) == keys diff --git a/uv.lock b/uv.lock index 62e3bd1..ca9d306 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,7 @@ members = [ "nbversion", "pyxray", "refcheck", + "wasmprobe", "xraymanim", "xraywidgets", ] @@ -333,6 +334,7 @@ dependencies = [ { name = "nbversion" }, { name = "pyxray" }, { name = "refcheck" }, + { name = "wasmprobe" }, { name = "xraymanim" }, { name = "xraywidgets" }, ] @@ -362,6 +364,7 @@ requires-dist = [ { name = "nbversion", editable = "tools/nbversion" }, { name = "pyxray", editable = "pyxray" }, { name = "refcheck", editable = "tools/refcheck" }, + { name = "wasmprobe", editable = "tools/wasmprobe" }, { name = "xraymanim", editable = "xraymanim" }, { name = "xraymanim", extras = ["render"], marker = "extra == 'anim'", editable = "xraymanim" }, { name = "xraywidgets", editable = "xraywidgets" }, @@ -1572,6 +1575,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] +[[package]] +name = "wasmprobe" +version = "0.1.0" +source = { editable = "tools/wasmprobe" } + [[package]] name = "watchdog" version = "6.0.0"