diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebddcdd..f760b64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,36 @@ jobs: # workspace copy is already installed, so what gets executed is the working tree and # not whatever is on main, which is the whole point of running them in a pull request. - run: uv run nbcheck run + # Same cells again, on this leg's interpreter, saved as one small JSON file per + # notebook. The versions job below is what reads them. Recording is separate from + # `nbcheck run` on purpose: that step stops at the first cell that raises, and this + # one keeps going, so a lesson that breaks cannot hide every version difference in + # the lessons after it. + - run: uv run nbversion record --into build/versions + - uses: actions/upload-artifact@v4 + with: + name: versions-${{ matrix.python }} + path: build/versions + retention-days: 1 + + # Both legs of the notebooks matrix ran the same cells on different interpreters. This + # job is the one that compares the two and fails if a cell's output depends on which + # Python the reader has and the lesson does not say so. + versions: + runs-on: ubuntu-latest + needs: notebooks + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - run: uv sync --all-packages + - uses: actions/download-artifact@v5 + with: + pattern: versions-* + merge-multiple: true + path: build/versions + - run: uv run nbversion compare build/versions/3.15 build/versions/3.14 blueprints: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ffc8d1..df60e7d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,6 +55,8 @@ lesson.code( Say what the reader is looking at and what the other version does instead. "This differs on 3.14" is not a note, it is an apology. Add `quiet=True` when a paragraph near the top of the lesson already explains a difference that then turns up in a dozen cells, so the same sentence is not repeated under every one of them. +There is a second keyword for the other kind of cell. `varies=` is for output that depends on the reader's machine rather than on the version: which flags their interpreter was configured with, how many files are in their standard library, how deep the C stack goes before it runs out. It reads exactly the same to a reader and the check treats it differently. A `differs` note is a claim two recordings can test, so it fails when it stops being true. A `varies` note is not, because whether two runs agree about a machine difference depends on which two machines made them, and a CI box where both interpreters came from the same builder would delete a note that is still right for somebody on a framework install. So `varies` is reported and never fails. Do not reach for it to silence a real version difference. + When the differing cell is the lesson's central observation, the note is not enough. Either the lesson gets a short section explaining both versions, because the difference is itself worth teaching, or the example changes to one that behaves the same on both. Which of the two depends on whether the difference is interesting. `LOAD_COMMON_CONSTANT` is interesting and gets explained. A line number inside `asyncio` is not, and the cell should stop printing it. ## Definition of done for a lesson diff --git a/lessons/t01-one-line-seven-stages/build.py b/lessons/t01-one-line-seven-stages/build.py index a6e47d4..a2e7642 100644 --- a/lessons/t01-one-line-seven-stages/build.py +++ b/lessons/t01-one-line-seven-stages/build.py @@ -15,7 +15,7 @@ image somebody finds later. """ -from nbbuild import Lesson +from nbbuild import BANNER, TRAILING_NONE, Lesson from nbdiagram import Diagrams lesson = Lesson("t01-one-line-seven-stages", "t01") @@ -76,14 +76,20 @@ ## Which interpreter is this Nearly every fact in this lesson is a fact about one particular build of CPython. Instruction names change between releases and so do the sizes of things, so every lesson here starts by saying out loud which binary is about to produce the output you are reading. + +If the banner says 3.14, which is what Colab installs today, nearly all of this lesson is the same and a handful of cells are not. The differences below all come from one change. On 3.15 the implicit `return None` at the end of a module is a `LOAD_COMMON_CONSTANT`, and `None` is not in the constant table at all. On 3.14 it is an ordinary `LOAD_CONST`, and `None` is in the table. That is one more constant and two fewer bytes of bytecode everywhere those get printed. Stage 6 comes back to it and asks your build directly. """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(""" @@ -247,9 +253,13 @@ """) -lesson.code(""" +lesson.code( + """ print(compiler.what_the_optimizer_did(result)) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -270,9 +280,13 @@ """) -lesson.code(r""" +lesson.code( + r""" print(compiler.what_the_optimizer_did(compiler.stages("answer = six * 7\n"))) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -282,7 +296,8 @@ """) -lesson.code(""" +lesson.code( + """ import dis print("co_consts ", result.code.co_consts) @@ -291,7 +306,10 @@ print("co_code ", len(result.code.co_code), "bytes") print() dis.dis(result.code) -""") +""", + differs="On 3.14 co_consts is (6, None) rather than (6,) and the bytecode is 10 bytes rather than 12, because None still needs a table entry there.", + quiet=True, +) lesson.md(f""" @@ -305,14 +323,18 @@ """) -lesson.code(""" +lesson.code( + """ loaded = {i.argval for i in dis.get_instructions(result.code) if i.opname == "LOAD_CONST"} inline = [str(i) for i in result.optimized if "SMALL_INT" in i.opname] print("constants the code object carries:", result.code.co_consts) print("constants any instruction loads: ", loaded or "none") print("where the 42 actually lives: ", inline) -""") +""", + differs="On 3.14 one instruction does load a constant, the None at the end, so the middle line says {None} rather than none. The point of the cell survives: nothing loads the 6.", + quiet=True, +) lesson.md(f""" @@ -322,13 +344,17 @@ """) -lesson.code(""" +lesson.code( + """ print("python ", sys.version.split()[0]) print("last instructions ", [str(item) for item in result.optimized[-2:]]) print("co_consts ", result.code.co_consts) print("bytecode ", len(result.code.co_code), "bytes") print("None is a constant", None in result.code.co_consts) -""") +""", + differs="This cell is here to differ. On 3.14 the last line says True and on 3.15 it says False, which is the whole paragraph above turned into output.", + quiet=True, +) lesson.md(f""" @@ -353,9 +379,13 @@ """) -lesson.code(""" +lesson.code( + """ print(result.summary()) -""") +""", + differs="On 3.14 the last number is 10 bytes rather than 12. Everything to the left of it is the same.", + quiet=True, +) lesson.md(""" @@ -382,14 +412,18 @@ """) -lesson.code(r""" +lesson.code( + r""" MINE = "x = 2 ** 10\n" mine = compiler.stages(MINE) print(mine.summary()) print() print(compiler.what_the_optimizer_did(mine)) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(""" diff --git a/lessons/t01-one-line-seven-stages/t01.ipynb b/lessons/t01-one-line-seven-stages/t01.ipynb index 074edad..e7aff20 100644 --- a/lessons/t01-one-line-seven-stages/t01.ipynb +++ b/lessons/t01-one-line-seven-stages/t01.ipynb @@ -69,14 +69,20 @@ "source": [ "## Which interpreter is this\n", "\n", - "Nearly every fact in this lesson is a fact about one particular build of CPython. Instruction names change between releases and so do the sizes of things, so every lesson here starts by saying out loud which binary is about to produce the output you are reading." + "Nearly every fact in this lesson is a fact about one particular build of CPython. Instruction names change between releases and so do the sizes of things, so every lesson here starts by saying out loud which binary is about to produce the output you are reading.\n", + "\n", + "If the banner says 3.14, which is what Colab installs today, nearly all of this lesson is the same and a handful of cells are not. The differences below all come from one change. On 3.15 the implicit `return None` at the end of a module is a `LOAD_COMMON_CONSTANT`, and `None` is not in the constant table at all. On 3.14 it is an ordinary `LOAD_CONST`, and `None` is in the table. That is one more constant and two fewer bytes of bytecode everywhere those get printed. Stage 6 comes back to it and asks your build directly." ] }, { "cell_type": "code", "execution_count": null, "id": "t01-06", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -325,7 +331,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-27", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "print(compiler.what_the_optimizer_did(result))" @@ -359,7 +369,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-30", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "print(compiler.what_the_optimizer_did(compiler.stages(\"answer = six * 7\\n\")))" @@ -379,7 +393,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-32", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 co_consts is (6, None) rather than (6,) and the bytecode is 10 bytes rather than 12, because None still needs a table entry there." + } + }, "outputs": [], "source": [ "import dis\n", @@ -410,7 +428,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-34", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 one instruction does load a constant, the None at the end, so the middle line says {None} rather than none. The point of the cell survives: nothing loads the 6." + } + }, "outputs": [], "source": [ "loaded = {i.argval for i in dis.get_instructions(result.code) if i.opname == \"LOAD_CONST\"}\n", @@ -435,7 +457,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-36", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This cell is here to differ. On 3.14 the last line says True and on 3.15 it says False, which is the whole paragraph above turned into output." + } + }, "outputs": [], "source": [ "print(\"python \", sys.version.split()[0])\n", @@ -482,7 +508,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-40", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the last number is 10 bytes rather than 12. Everything to the left of it is the same." + } + }, "outputs": [], "source": [ "print(result.summary())" @@ -522,7 +552,11 @@ "cell_type": "code", "execution_count": null, "id": "t01-43", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "MINE = \"x = 2 ** 10\\n\"\n", diff --git a/lessons/t02-text-becomes-tokens/build.py b/lessons/t02-text-becomes-tokens/build.py index 31656c1..c8e2b7b 100644 --- a/lessons/t02-text-becomes-tokens/build.py +++ b/lessons/t02-text-becomes-tokens/build.py @@ -16,7 +16,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, Lesson from nbdiagram import Diagrams lesson = Lesson("t02-text-becomes-tokens", "t02") @@ -92,11 +92,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" diff --git a/lessons/t02-text-becomes-tokens/t02.ipynb b/lessons/t02-text-becomes-tokens/t02.ipynb index 7926f3b..8e57d2e 100644 --- a/lessons/t02-text-becomes-tokens/t02.ipynb +++ b/lessons/t02-text-becomes-tokens/t02.ipynb @@ -88,7 +88,11 @@ "cell_type": "code", "execution_count": null, "id": "t02-06", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", diff --git a/lessons/t03-tokens-become-a-tree/build.py b/lessons/t03-tokens-become-a-tree/build.py index 153e113..98f8d3a 100644 --- a/lessons/t03-tokens-become-a-tree/build.py +++ b/lessons/t03-tokens-become-a-tree/build.py @@ -22,7 +22,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, YOUR_INSTALL, Lesson from nbdiagram import Diagrams lesson = Lesson("t03-tokens-become-a-tree", "t03") @@ -90,11 +90,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" @@ -287,12 +291,15 @@ """) -lesson.code(""" +lesson.code( + """ report = trees.survey(trees.stdlib()) print("standard library at", trees.stdlib()) print(report) -""") +""", + varies=YOUR_INSTALL, +) lesson.md(""" diff --git a/lessons/t03-tokens-become-a-tree/t03.ipynb b/lessons/t03-tokens-become-a-tree/t03.ipynb index ce41403..c91dd82 100644 --- a/lessons/t03-tokens-become-a-tree/t03.ipynb +++ b/lessons/t03-tokens-become-a-tree/t03.ipynb @@ -80,7 +80,11 @@ "cell_type": "code", "execution_count": null, "id": "t03-06", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -369,7 +373,11 @@ "cell_type": "code", "execution_count": null, "id": "t03-30", - "metadata": {}, + "metadata": { + "cpython_internals": { + "varies": "These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." + } + }, "outputs": [], "source": [ "report = trees.survey(trees.stdlib())\n", @@ -382,6 +390,14 @@ "cell_type": "markdown", "id": "t03-31", "metadata": {}, + "source": [ + "> **Version note.** These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." + ] + }, + { + "cell_type": "markdown", + "id": "t03-32", + "metadata": {}, "source": [ "Every module gives the same tree both times. That is a real property test over a few hundred thousand lines of code that nobody wrote for this lesson, and it took about a second.\n", "\n", @@ -390,7 +406,7 @@ }, { "cell_type": "markdown", - "id": "t03-32", + "id": "t03-33", "metadata": {}, "source": [ "## Try it yourself\n", @@ -411,7 +427,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t03-33", + "id": "t03-34", "metadata": {}, "outputs": [], "source": [ @@ -424,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "t03-34", + "id": "t03-35", "metadata": {}, "source": [ "## What just happened\n", @@ -436,7 +452,7 @@ }, { "cell_type": "markdown", - "id": "t03-35", + "id": "t03-36", "metadata": {}, "source": [ "## Where this goes next\n", diff --git a/lessons/t04-names-get-scopes/build.py b/lessons/t04-names-get-scopes/build.py index 0bf6d8a..9314424 100644 --- a/lessons/t04-names-get-scopes/build.py +++ b/lessons/t04-names-get-scopes/build.py @@ -20,7 +20,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, Lesson from nbdiagram import Diagrams lesson = Lesson("t04-names-get-scopes", "t04") @@ -94,11 +94,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(""" diff --git a/lessons/t04-names-get-scopes/t04.ipynb b/lessons/t04-names-get-scopes/t04.ipynb index c917056..0857dca 100644 --- a/lessons/t04-names-get-scopes/t04.ipynb +++ b/lessons/t04-names-get-scopes/t04.ipynb @@ -86,7 +86,11 @@ "cell_type": "code", "execution_count": null, "id": "t04-06", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", diff --git a/lessons/t05-the-tree-becomes-bytecode/build.py b/lessons/t05-the-tree-becomes-bytecode/build.py index 25dd083..017464c 100644 --- a/lessons/t05-the-tree-becomes-bytecode/build.py +++ b/lessons/t05-the-tree-becomes-bytecode/build.py @@ -20,7 +20,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, TRAILING_NONE, Lesson from nbdiagram import Diagrams lesson = Lesson("t05-the-tree-becomes-bytecode", "t05") @@ -91,14 +91,20 @@ ## Which Python is this More of this lesson is version dependent than usual. The optimizer is where CPython's release notes spend most of their "faster" bullet points, so the exact instruction list below is the one your interpreter produced and not the one somebody wrote down in 2023. + +Everything below was checked against the version this cell prints and against 3.14, which is what Colab installs today. Two differences account for nearly all of the disagreements. On 3.15 the implicit `return None` at the end of a module is a `LOAD_COMMON_CONSTANT` and `None` is not in the constant table at all, where on 3.14 it is an ordinary `LOAD_CONST` and `None` is in the table. And on 3.15 `RESUME` and `GET_ITER` carry an inline cache, where on 3.14 they do not. So if you are on 3.14 you will see one more constant, two fewer bytes of bytecode, and every offset two to four lower than the numbers in the text. The shape of every listing is the same. """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" @@ -132,11 +138,15 @@ """) -lesson.code(""" +lesson.code( + """ result = compiler.stages("answer = 6 * 7") print(result.summary()) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(""" @@ -148,9 +158,13 @@ """) -lesson.code(""" +lesson.code( + """ print(compiler.what_the_optimizer_did(result)) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -175,11 +189,15 @@ """) -lesson.code(""" +lesson.code( + """ code = compile("answer = 6 * 7", "lesson.py", "exec") print("co_consts:", code.co_consts) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -193,13 +211,17 @@ """) -lesson.code(""" +lesson.code( + """ import dis used = [step.arg for step in dis.get_instructions(code) if step.opname == "LOAD_CONST"] print("constants in the table:", code.co_consts) print("constants actually loaded:", used) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -224,7 +246,8 @@ """) -lesson.code(''' +lesson.code( + ''' guarded = """ try: n = 1 / 0 @@ -233,7 +256,10 @@ """ print(compiler.what_the_optimizer_did(compiler.stages(guarded))) -''') +''', + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -251,7 +277,8 @@ """) -lesson.code(''' +lesson.code( + ''' never = """ if False: print("this never runs") @@ -259,7 +286,10 @@ """ print(compiler.what_the_optimizer_did(compiler.stages(never))) -''') +''', + differs=TRAILING_NONE, + quiet=True, +) lesson.md(f""" @@ -273,9 +303,13 @@ """) -lesson.code(""" +lesson.code( + """ print(compile(never, "lesson.py", "exec").co_consts) -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(""" @@ -289,7 +323,8 @@ """) -lesson.code(''' +lesson.code( + ''' looping = """ total = 0 for n in [1, 2, 3]: @@ -297,7 +332,10 @@ """ print(compiler.what_the_optimizer_did(compiler.stages(looping))) -''') +''', + differs="On 3.14 the trailing None is a LOAD_CONST, GET_ITER prints without an argument, and the constant indices shift by one.", + quiet=True, +) lesson.md(f""" @@ -374,9 +412,13 @@ """) -lesson.code(""" +lesson.code( + """ print(compile("answer = 6 * 7", "lesson.py", "exec").co_code.hex(" ")) -""") +""", + differs="On 3.14 this is 10 bytes rather than 12, and the bytes themselves are different, because RESUME has no inline cache and the final None is a LOAD_CONST.", + quiet=True, +) lesson.md(f""" @@ -396,12 +438,16 @@ """) -lesson.code(""" +lesson.code( + """ from pyxray import bytecode for start, end, line in bytecode.line_table("answer = 6 * 7"): print(f"bytes {start:>3} to {end:<3} came from line {line}") -""") +""", + differs="On 3.14 the ranges start two bytes lower, because RESUME has no inline cache there.", + quiet=True, +) lesson.md(f""" @@ -409,7 +455,8 @@ """) -lesson.code(''' +lesson.code( + ''' faulty = """ try: n = 1 / 0 @@ -420,7 +467,10 @@ table = compile(faulty, "lesson.py", "exec").co_exceptiontable print("exception table, in bytes:", len(table)) print(table.hex(" ")) -''') +''', + differs="On 3.14 the table is 12 bytes rather than 16, and the bytes are different. It encodes offsets into the bytecode, and those offsets are different on the two versions.", + quiet=True, +) lesson.md(""" @@ -438,7 +488,8 @@ """) -lesson.code(''' +lesson.code( + ''' experiment = """ x = 1 y = 2 @@ -446,7 +497,10 @@ """ print(compiler.what_the_optimizer_did(compiler.stages(experiment))) -''') +''', + differs=TRAILING_NONE, + quiet=True, +) lesson.md(""" diff --git a/lessons/t05-the-tree-becomes-bytecode/t05.ipynb b/lessons/t05-the-tree-becomes-bytecode/t05.ipynb index 912bf60..c57be8f 100644 --- a/lessons/t05-the-tree-becomes-bytecode/t05.ipynb +++ b/lessons/t05-the-tree-becomes-bytecode/t05.ipynb @@ -79,14 +79,20 @@ "source": [ "## Which Python is this\n", "\n", - "More of this lesson is version dependent than usual. The optimizer is where CPython's release notes spend most of their \"faster\" bullet points, so the exact instruction list below is the one your interpreter produced and not the one somebody wrote down in 2023." + "More of this lesson is version dependent than usual. The optimizer is where CPython's release notes spend most of their \"faster\" bullet points, so the exact instruction list below is the one your interpreter produced and not the one somebody wrote down in 2023.\n", + "\n", + "Everything below was checked against the version this cell prints and against 3.14, which is what Colab installs today. Two differences account for nearly all of the disagreements. On 3.15 the implicit `return None` at the end of a module is a `LOAD_COMMON_CONSTANT` and `None` is not in the constant table at all, where on 3.14 it is an ordinary `LOAD_CONST` and `None` is in the table. And on 3.15 `RESUME` and `GET_ITER` carry an inline cache, where on 3.14 they do not. So if you are on 3.14 you will see one more constant, two fewer bytes of bytecode, and every offset two to four lower than the numbers in the text. The shape of every listing is the same." ] }, { "cell_type": "code", "execution_count": null, "id": "t05-06", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -140,7 +146,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-10", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "result = compiler.stages(\"answer = 6 * 7\")\n", @@ -164,7 +174,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-12", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "print(compiler.what_the_optimizer_did(result))" @@ -202,7 +216,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-15", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "code = compile(\"answer = 6 * 7\", \"lesson.py\", \"exec\")\n", @@ -228,7 +246,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-17", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "import dis\n", @@ -275,7 +297,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-21", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "guarded = \"\"\"\n", @@ -310,7 +336,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-23", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "never = \"\"\"\n", @@ -340,7 +370,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-25", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "print(compile(never, \"lesson.py\", \"exec\").co_consts)" @@ -364,7 +398,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-27", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the trailing None is a LOAD_CONST, GET_ITER prints without an argument, and the constant indices shift by one." + } + }, "outputs": [], "source": [ "looping = \"\"\"\n", @@ -484,7 +522,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-36", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 this is 10 bytes rather than 12, and the bytes themselves are different, because RESUME has no inline cache and the final None is a LOAD_CONST." + } + }, "outputs": [], "source": [ "print(compile(\"answer = 6 * 7\", \"lesson.py\", \"exec\").co_code.hex(\" \"))" @@ -514,7 +556,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-38", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the ranges start two bytes lower, because RESUME has no inline cache there." + } + }, "outputs": [], "source": [ "from pyxray import bytecode\n", @@ -535,7 +581,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-40", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the table is 12 bytes rather than 16, and the bytes are different. It encodes offsets into the bytecode, and those offsets are different on the two versions." + } + }, "outputs": [], "source": [ "faulty = \"\"\"\n", @@ -572,7 +622,11 @@ "cell_type": "code", "execution_count": null, "id": "t05-42", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "experiment = \"\"\"\n", diff --git a/lessons/t06-reading-bytecode-fluently/build.py b/lessons/t06-reading-bytecode-fluently/build.py index dee7e67..52334be 100644 --- a/lessons/t06-reading-bytecode-fluently/build.py +++ b/lessons/t06-reading-bytecode-fluently/build.py @@ -20,7 +20,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, OFFSETS, YOUR_INSTALL, Lesson from nbdiagram import Diagrams lesson = Lesson("t06-reading-bytecode-fluently", "t06") @@ -82,14 +82,20 @@ ## Which Python is this Instruction names change between releases more than almost anything else in CPython. Half the opcodes in the listings below did not exist three releases ago. Everything here was checked against the version this cell prints, and if yours is different the shapes will still be right even where the names are not. + +It was also checked against 3.14, which is what Colab installs today, and one difference shows up in almost every listing. On 3.15 `RESUME` and `GET_ITER` carry an inline cache slot and on 3.14 they do not, so on 3.14 every offset below is two to four lower than the number in the text. Nothing else about the listing changes. Where a cell differs for some other reason, it says so underneath. """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" @@ -107,11 +113,15 @@ """) -lesson.code(""" +lesson.code( + """ from pyxray import stack print(stack.table("total = 0\\ntotal = total + 1")) -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(""" @@ -136,7 +146,8 @@ """) -lesson.code(""" +lesson.code( + """ import opcode code = compile("print(total)", "lesson.py", "exec") @@ -147,7 +158,10 @@ for offset in range(0, 8, 2): opcode_byte, argument_byte = code.co_code[offset], code.co_code[offset + 1] print(f"{offset:>3} {opcode_byte:>3} {argument_byte:>3} {opcode.opname[opcode_byte]}") -""") +""", + differs="On 3.14 RESUME has no cache entry, so the second row is LOAD_NAME rather than CACHE and every offset after it is two lower.", + quiet=True, +) lesson.md(f""" @@ -178,7 +192,8 @@ """) -lesson.code(""" +lesson.code( + """ def greet(name): return "hello " + name @@ -187,7 +202,10 @@ def greet(name): meaning = "" if item.arg is None else bytecode.argument_meaning(item.opname) argument = "" if item.arg is None else str(item.arg) print(f"{item.offset:>4} {item.opname:<20} {argument:>4} {meaning}") -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(f""" @@ -229,7 +247,8 @@ def two_at_once(first, second, third, fourth): """) -lesson.code(""" +lesson.code( + """ import dis crowded = "\\n".join(f"a{i} = {i}" for i in range(300)) + "\\nprint(a256)\\n" @@ -243,7 +262,10 @@ def two_at_once(first, second, third, fourth): print(f"{item.arg} * 256 + {following.arg % 256} = {following.arg}") print("co_names[" + str(following.arg) + "] is", code.co_names[following.arg]) break -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(""" @@ -266,7 +288,8 @@ def two_at_once(first, second, third, fourth): """) -lesson.code(''' +lesson.code( + ''' loop = """ total = 0 for n in [1, 2, 3]: @@ -275,7 +298,10 @@ def two_at_once(first, second, third, fourth): """ print(bytecode.table(loop, show_caches=True)) -''') +''', + differs=OFFSETS, + quiet=True, +) lesson.md(f""" @@ -293,9 +319,13 @@ def two_at_once(first, second, third, fourth): """) -lesson.code(""" +lesson.code( + """ print(bytecode.jump_table(loop)) -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(""" @@ -331,12 +361,15 @@ def two_at_once(first, second, third, fourth): """) -lesson.code(""" +lesson.code( + """ for name in ["GET_ITER", "BINARY_OP", "END_FOR", "POP_ITER"]: number = opcode.opmap[name] argument = 0 if number >= opcode.HAVE_ARGUMENT else None print(f"{name:<12} {dis.stack_effect(number, argument):>3}") -""") +""", + differs="On 3.14 GET_ITER is 0 and POP_ITER is -1. In 3.15 GET_ITER leaves one more item on the stack than it takes, and POP_ITER takes that extra item away again.", +) lesson.md(f""" @@ -352,7 +385,8 @@ def two_at_once(first, second, third, fourth): """) -lesson.code(''' +lesson.code( + ''' guarded = """ try: value = int("x") @@ -361,7 +395,10 @@ def two_at_once(first, second, third, fourth): """ print(stack.table(guarded)) -''') +''', + differs=OFFSETS, + quiet=True, +) lesson.md(""" @@ -375,7 +412,8 @@ def two_at_once(first, second, third, fourth): """) -lesson.code(""" +lesson.code( + """ import sysconfig import types from pathlib import Path @@ -403,7 +441,9 @@ def every_code_object(code): disagreed += stack.high_water(code_object) != code_object.co_stacksize print(f"{len(files)} files, {checked} code objects, {disagreed} disagreements") -""") +""", + varies=YOUR_INSTALL, +) lesson.md(""" @@ -490,7 +530,8 @@ def only_raises(): """) -lesson.code(""" +lesson.code( + """ def add_up_the_positive_ones(items): total = 0 for item in items: @@ -500,7 +541,10 @@ def add_up_the_positive_ones(items): print(bytecode.table(add_up_the_positive_ones)) -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(""" diff --git a/lessons/t06-reading-bytecode-fluently/t06.ipynb b/lessons/t06-reading-bytecode-fluently/t06.ipynb index 0adc942..bd0b6bc 100644 --- a/lessons/t06-reading-bytecode-fluently/t06.ipynb +++ b/lessons/t06-reading-bytecode-fluently/t06.ipynb @@ -67,14 +67,20 @@ "source": [ "## Which Python is this\n", "\n", - "Instruction names change between releases more than almost anything else in CPython. Half the opcodes in the listings below did not exist three releases ago. Everything here was checked against the version this cell prints, and if yours is different the shapes will still be right even where the names are not." + "Instruction names change between releases more than almost anything else in CPython. Half the opcodes in the listings below did not exist three releases ago. Everything here was checked against the version this cell prints, and if yours is different the shapes will still be right even where the names are not.\n", + "\n", + "It was also checked against 3.14, which is what Colab installs today, and one difference shows up in almost every listing. On 3.15 `RESUME` and `GET_ITER` carry an inline cache slot and on 3.14 they do not, so on 3.14 every offset below is two to four lower than the number in the text. Nothing else about the listing changes. Where a cell differs for some other reason, it says so underneath." ] }, { "cell_type": "code", "execution_count": null, "id": "t06-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -104,7 +110,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-07", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "from pyxray import stack\n", @@ -144,7 +154,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-10", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME has no cache entry, so the second row is LOAD_NAME rather than CACHE and every offset after it is two lower." + } + }, "outputs": [], "source": [ "import opcode\n", @@ -202,7 +216,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-14", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "def greet(name):\n", @@ -269,7 +287,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-18", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "import dis\n", @@ -317,7 +339,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-21", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "loop = \"\"\"\n", @@ -352,7 +378,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-23", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "print(bytecode.jump_table(loop))" @@ -406,7 +436,11 @@ "cell_type": "code", "execution_count": null, "id": "t06-27", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 GET_ITER is 0 and POP_ITER is -1. In 3.15 GET_ITER leaves one more item on the stack than it takes, and POP_ITER takes that extra item away again." + } + }, "outputs": [], "source": [ "for name in [\"GET_ITER\", \"BINARY_OP\", \"END_FOR\", \"POP_ITER\"]:\n", @@ -419,6 +453,14 @@ "cell_type": "markdown", "id": "t06-28", "metadata": {}, + "source": [ + "> **Version note.** On 3.14 GET_ITER is 0 and POP_ITER is -1. In 3.15 GET_ITER leaves one more item on the stack than it takes, and POP_ITER takes that extra item away again." + ] + }, + { + "cell_type": "markdown", + "id": "t06-29", + "metadata": {}, "source": [ "`POP_ITER` takes two off. That is the iterator and the value under it, which is how a `for` loop tidies up after itself.\n", "\n", @@ -434,8 +476,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t06-29", - "metadata": {}, + "id": "t06-30", + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "guarded = \"\"\"\n", @@ -450,7 +496,7 @@ }, { "cell_type": "markdown", - "id": "t06-30", + "id": "t06-31", "metadata": {}, "source": [ "Look at the row where the handler starts. The instruction above it ended at one height and this one begins at a different one, which looks like a mistake until you remember the listing is in address order and nothing falls into a handler from above. The only way to reach it is to raise.\n", @@ -465,8 +511,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t06-31", - "metadata": {}, + "id": "t06-32", + "metadata": { + "cpython_internals": { + "varies": "These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." + } + }, "outputs": [], "source": [ "import sysconfig\n", @@ -500,7 +550,15 @@ }, { "cell_type": "markdown", - "id": "t06-32", + "id": "t06-33", + "metadata": {}, + "source": [ + "> **Version note.** These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." + ] + }, + { + "cell_type": "markdown", + "id": "t06-34", "metadata": {}, "source": [ "It found no disagreements. Run it over the whole standard library rather than the first 150 files and it still finds none, across thirty three thousand code objects.\n", @@ -515,7 +573,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t06-33", + "id": "t06-35", "metadata": {}, "outputs": [], "source": [ @@ -529,7 +587,7 @@ }, { "cell_type": "markdown", - "id": "t06-34", + "id": "t06-36", "metadata": {}, "source": [ "Nothing is ever pushed in that function, so the honest answer is zero, and CPython says one.\n", @@ -550,7 +608,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t06-35", + "id": "t06-37", "metadata": {}, "outputs": [], "source": [ @@ -583,7 +641,7 @@ }, { "cell_type": "markdown", - "id": "t06-36", + "id": "t06-38", "metadata": {}, "source": [ "Work through it with the four questions.\n", @@ -604,8 +662,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t06-37", - "metadata": {}, + "id": "t06-39", + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "def add_up_the_positive_ones(items):\n", @@ -621,7 +683,7 @@ }, { "cell_type": "markdown", - "id": "t06-38", + "id": "t06-40", "metadata": {}, "source": [ "One row in there is worth a second look. `RETURN_VALUE` has a stack effect of zero, which cannot be right for an instruction whose whole job is to hand a value back. It is right because the frame goes away at the same moment, and the value is pushed onto the caller's stack rather than removed from this one. The tables describe the frame you are looking at.\n", @@ -639,7 +701,7 @@ }, { "cell_type": "markdown", - "id": "t06-39", + "id": "t06-41", "metadata": {}, "source": [ "## What just happened\n", diff --git a/lessons/t07-the-machine-runs/build.py b/lessons/t07-the-machine-runs/build.py index a12de28..aecd8fe 100644 --- a/lessons/t07-the-machine-runs/build.py +++ b/lessons/t07-the-machine-runs/build.py @@ -18,7 +18,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, OFFSETS, Lesson from nbdiagram import Diagrams lesson = Lesson("t07-the-machine-runs", "t07") @@ -80,14 +80,20 @@ ## Which Python is this Instruction names and monitoring events both change between releases. Everything here was checked against the version this cell prints. + +It was also checked against 3.14, which is what Colab installs today, and one difference shows up in almost every listing. On 3.15 `RESUME` and `GET_ITER` carry an inline cache slot and on 3.14 they do not, so on 3.14 every offset below is two to four lower than the number in the text. Nothing else about the listing changes. Where a cell differs for some other reason, it says so underneath. """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" @@ -122,7 +128,8 @@ """) -lesson.code(""" +lesson.code( + """ import sysconfig arguments = sysconfig.get_config_var("CONFIG_ARGS") or "" @@ -135,7 +142,9 @@ for argument in arguments.replace("'", "").split(): if argument.startswith("--with") or argument.startswith("--enable"): print(" ", argument) -""") +""", + varies="This lists the flags your CPython was configured with, which says more about who built it than about the version.", +) lesson.md(""" @@ -188,7 +197,8 @@ """) -lesson.code(""" +lesson.code( + """ import sys sys.setrecursionlimit(200_000) @@ -216,7 +226,9 @@ def through_c(n): except RecursionError as problem: print(f"through sorted, gave up somewhere under {depth} deep") print(" ", problem) -""") +""", + varies="How deep you get before the C stack runs out depends on the build and on the machine, so this number is different everywhere. That it stops far earlier than the pure Python version is the part to read.", +) lesson.md(f""" @@ -337,7 +349,8 @@ def unwound(code, offset, exception): """) -lesson.code(""" +lesson.code( + """ import sys monitoring = sys.monitoring @@ -366,7 +379,10 @@ def nothing(): print(f"{len(local)} events can be turned on for a single code object:") print(" ", ", ".join(local)) -""") +""", + differs="3.14 has 12 of these events rather than 17. EXCEPTION_HANDLED, PY_THROW, PY_UNWIND, RAISE and RERAISE cannot be set on a single code object there.", + quiet=True, +) lesson.md(""" @@ -387,7 +403,8 @@ def nothing(): """) -lesson.code(""" +lesson.code( + """ from pyxray import stepper @@ -404,7 +421,9 @@ def total_of(items): print("co_stacksize says:", total_of.__code__.co_stacksize) print() print(recording.table()) -""") +""", + differs="On 3.14 the offsets are lower and the deepest the stack gets is 3 rather than 4, because the loop is compiled a little differently. The shape of the recording is the same.", +) lesson.md(""" @@ -429,7 +448,8 @@ def total_of(items): """) -lesson.code(""" +lesson.code( + """ import dis compiled = {item.offset: item.opname for item in dis.get_instructions(total_of)} @@ -439,7 +459,10 @@ def total_of(items): for offset, opname in compiled.items(): if offset not in executed: print(f" {offset:>4} {opname}") -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(""" @@ -449,7 +472,8 @@ def total_of(items): """) -lesson.code(""" +lesson.code( + """ import sys monitoring = sys.monitoring @@ -480,7 +504,10 @@ def callback(code, offset, destination): finally: monitoring.set_local_events(3, total_of.__code__, 0) monitoring.free_tool_id(3) -""") +""", + differs=OFFSETS, + quiet=True, +) lesson.md(""" @@ -651,7 +678,8 @@ def show(): """) -lesson.code(""" +lesson.code( + """ from pyxray import stepper @@ -669,7 +697,9 @@ def first(): first() -""") +""", + varies="The frames above yours belong to whatever is running the notebook, so those names and line numbers come from Jupyter and asyncio rather than from anything the lesson did. The bottom of the list is your three functions, and that is the part to read.", +) lesson.md(""" diff --git a/lessons/t07-the-machine-runs/t07.ipynb b/lessons/t07-the-machine-runs/t07.ipynb index f417d6b..b7e526a 100644 --- a/lessons/t07-the-machine-runs/t07.ipynb +++ b/lessons/t07-the-machine-runs/t07.ipynb @@ -67,14 +67,20 @@ "source": [ "## Which Python is this\n", "\n", - "Instruction names and monitoring events both change between releases. Everything here was checked against the version this cell prints." + "Instruction names and monitoring events both change between releases. Everything here was checked against the version this cell prints.\n", + "\n", + "It was also checked against 3.14, which is what Colab installs today, and one difference shows up in almost every listing. On 3.15 `RESUME` and `GET_ITER` carry an inline cache slot and on 3.14 they do not, so on 3.14 every offset below is two to four lower than the number in the text. Nothing else about the listing changes. Where a cell differs for some other reason, it says so underneath." ] }, { "cell_type": "code", "execution_count": null, "id": "t07-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -124,7 +130,11 @@ "cell_type": "code", "execution_count": null, "id": "t07-08", - "metadata": {}, + "metadata": { + "cpython_internals": { + "varies": "This lists the flags your CPython was configured with, which says more about who built it than about the version." + } + }, "outputs": [], "source": [ "import sysconfig\n", @@ -146,13 +156,21 @@ "id": "t07-09", "metadata": {}, "source": [ - "Same instruction set and the same results either way. This is one of the places where CPython is really two or three programs built from the same source, and the difference is invisible from Python except through the note above." + "> **Version note.** This lists the flags your CPython was configured with, which says more about who built it than about the version." ] }, { "cell_type": "markdown", "id": "t07-10", "metadata": {}, + "source": [ + "Same instruction set and the same results either way. This is one of the places where CPython is really two or three programs built from the same source, and the difference is invisible from Python except through the note above." + ] + }, + { + "cell_type": "markdown", + "id": "t07-11", + "metadata": {}, "source": [ "## What one call needs\n", "\n", @@ -169,7 +187,7 @@ }, { "cell_type": "markdown", - "id": "t07-11", + "id": "t07-12", "metadata": {}, "source": [ "## Calling Python from Python\n", @@ -187,7 +205,7 @@ }, { "cell_type": "markdown", - "id": "t07-12", + "id": "t07-13", "metadata": {}, "source": [ "## Two stacks, and only one of them is small\n", @@ -206,8 +224,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-13", - "metadata": {}, + "id": "t07-14", + "metadata": { + "cpython_internals": { + "varies": "How deep you get before the C stack runs out depends on the build and on the machine, so this number is different everywhere. That it stops far earlier than the pure Python version is the part to read." + } + }, "outputs": [], "source": [ "import sys\n", @@ -241,7 +263,15 @@ }, { "cell_type": "markdown", - "id": "t07-14", + "id": "t07-15", + "metadata": {}, + "source": [ + "> **Version note.** How deep you get before the C stack runs out depends on the build and on the machine, so this number is different everywhere. That it stops far earlier than the pure Python version is the part to read." + ] + }, + { + "cell_type": "markdown", + "id": "t07-16", "metadata": {}, "source": [ "The message says \"Stack overflow\" and gives a size in kilobytes, which is CPython telling you it ran out of C stack rather than out of its own recursion budget. The limit you set with `setrecursionlimit` was never reached.\n", @@ -253,7 +283,7 @@ }, { "cell_type": "markdown", - "id": "t07-15", + "id": "t07-17", "metadata": {}, "source": [ "## Watching it happen\n", @@ -272,7 +302,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-16", + "id": "t07-18", "metadata": {}, "outputs": [], "source": [ @@ -292,7 +322,7 @@ }, { "cell_type": "markdown", - "id": "t07-17", + "id": "t07-19", "metadata": {}, "source": [ "## Frames appearing and disappearing\n", @@ -303,7 +333,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-18", + "id": "t07-20", "metadata": {}, "outputs": [], "source": [ @@ -369,7 +399,7 @@ }, { "cell_type": "markdown", - "id": "t07-19", + "id": "t07-21", "metadata": {}, "source": [ "Three `leaf` frames go on, the innermost one raises, and all three come off through `PY_UNWIND` rather than `PY_RETURN`. Then `top` catches it and returns normally. That is the frame stack unwinding, one frame per line, as it happens.\n", @@ -382,8 +412,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-20", - "metadata": {}, + "id": "t07-22", + "metadata": { + "cpython_internals": { + "differs": "3.14 has 12 of these events rather than 17. EXCEPTION_HANDLED, PY_THROW, PY_UNWIND, RAISE and RERAISE cannot be set on a single code object there." + } + }, "outputs": [], "source": [ "import sys\n", @@ -418,7 +452,7 @@ }, { "cell_type": "markdown", - "id": "t07-21", + "id": "t07-23", "metadata": {}, "source": [ "## One instruction at a time\n", @@ -428,7 +462,7 @@ }, { "cell_type": "markdown", - "id": "t07-22", + "id": "t07-24", "metadata": {}, "source": [ "![static heights and observed order joined into one listing](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t07-the-machine-runs/diagrams/where-the-numbers-come-from.svg)\n", @@ -443,8 +477,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-23", - "metadata": {}, + "id": "t07-25", + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the offsets are lower and the deepest the stack gets is 3 rather than 4, because the loop is compiled a little differently. The shape of the recording is the same." + } + }, "outputs": [], "source": [ "from pyxray import stepper\n", @@ -467,7 +505,15 @@ }, { "cell_type": "markdown", - "id": "t07-24", + "id": "t07-26", + "metadata": {}, + "source": [ + "> **Version note.** On 3.14 the offsets are lower and the deepest the stack gets is 3 rather than 4, because the loop is compiled a little differently. The shape of the recording is the same." + ] + }, + { + "cell_type": "markdown", + "id": "t07-27", "metadata": {}, "source": [ "Read the offset column rather than the step column. It counts up, then drops back to the offset of the `FOR_ITER`, three times over. That is the loop, and the drop is the back edge T06 taught you to spot in a listing, here being taken.\n", @@ -481,7 +527,7 @@ }, { "cell_type": "markdown", - "id": "t07-25", + "id": "t07-28", "metadata": {}, "source": [ "### The instruction that never shows up\n", @@ -496,8 +542,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-26", - "metadata": {}, + "id": "t07-29", + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "import dis\n", @@ -513,7 +563,7 @@ }, { "cell_type": "markdown", - "id": "t07-27", + "id": "t07-30", "metadata": {}, "source": [ "## Which way did the branch go\n", @@ -524,8 +574,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-28", - "metadata": {}, + "id": "t07-31", + "metadata": { + "cpython_internals": { + "differs": "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + } + }, "outputs": [], "source": [ "import sys\n", @@ -562,7 +616,7 @@ }, { "cell_type": "markdown", - "id": "t07-29", + "id": "t07-32", "metadata": {}, "source": [ "Five lines for a two item loop. Every `BRANCH_LEFT` and `BRANCH_RIGHT` is at the same offset, and that offset is the `FOR_ITER`. Left means there was another item, right means there was not. The `JUMP` is the back edge, taken once per item except the last.\n", @@ -576,7 +630,7 @@ }, { "cell_type": "markdown", - "id": "t07-30", + "id": "t07-33", "metadata": {}, "source": [ "![returning None keeps firing, returning DISABLE stops at that location](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t07-the-machine-runs/diagrams/turning-an-event-off.svg)\n", @@ -589,7 +643,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-31", + "id": "t07-34", "metadata": {}, "outputs": [], "source": [ @@ -631,7 +685,7 @@ }, { "cell_type": "markdown", - "id": "t07-32", + "id": "t07-35", "metadata": {}, "source": [ "Forty calls against fifteen: the loop body ran five times and the callback saw it once.\n", @@ -644,7 +698,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-33", + "id": "t07-36", "metadata": {}, "outputs": [], "source": [ @@ -681,7 +735,7 @@ }, { "cell_type": "markdown", - "id": "t07-34", + "id": "t07-37", "metadata": {}, "source": [ "Thirty nine calls, with no way to reduce them except by turning the whole thing off. `sys.monitoring` was added because debuggers and coverage tools were paying that price on every line of every program they touched.\n", @@ -698,7 +752,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-35", + "id": "t07-38", "metadata": {}, "outputs": [], "source": [ @@ -719,7 +773,7 @@ }, { "cell_type": "markdown", - "id": "t07-36", + "id": "t07-39", "metadata": {}, "source": [ "The frame object outliving the call is the whole reason frames are not on the C stack. A traceback holds onto frames, a generator is a frame that got paused, and a closure can keep one alive indefinitely. None of that would work if the frame went away when the C function returned.\n", @@ -730,7 +784,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-37", + "id": "t07-40", "metadata": {}, "outputs": [], "source": [ @@ -756,7 +810,7 @@ }, { "cell_type": "markdown", - "id": "t07-38", + "id": "t07-41", "metadata": {}, "source": [ "`f_locals` is a live view onto the frame's slots, so writing through it changes the actual local. `locals()` inside a function is a plain dictionary copied out of those slots, so writing to it changes nothing. This used to be much more confusing than it is now, and the current behaviour was pinned down deliberately.\n", @@ -769,8 +823,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t07-39", - "metadata": {}, + "id": "t07-42", + "metadata": { + "cpython_internals": { + "varies": "The frames above yours belong to whatever is running the notebook, so those names and line numbers come from Jupyter and asyncio rather than from anything the lesson did. The bottom of the list is your three functions, and that is the part to read." + } + }, "outputs": [], "source": [ "from pyxray import stepper\n", @@ -794,7 +852,15 @@ }, { "cell_type": "markdown", - "id": "t07-40", + "id": "t07-43", + "metadata": {}, + "source": [ + "> **Version note.** The frames above yours belong to whatever is running the notebook, so those names and line numbers come from Jupyter and asyncio rather than from anything the lesson did. The bottom of the list is your three functions, and that is the part to read." + ] + }, + { + "cell_type": "markdown", + "id": "t07-44", "metadata": {}, "source": [ "Innermost first, out to whatever is running the notebook. `stepper.chain` is nine lines and does nothing clever: take `sys._getframe()` and read `f_back` until it is `None`.\n", @@ -812,7 +878,7 @@ }, { "cell_type": "markdown", - "id": "t07-41", + "id": "t07-45", "metadata": {}, "source": [ "## What just happened\n", diff --git a/lessons/t08-everything-is-an-object/build.py b/lessons/t08-everything-is-an-object/build.py index a5a5ca4..cb2bd4d 100644 --- a/lessons/t08-everything-is-an-object/build.py +++ b/lessons/t08-everything-is-an-object/build.py @@ -17,7 +17,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, SMALL_INTS, TRAILING_NONE, Lesson from nbdiagram import Diagrams lesson = Lesson("t08-everything-is-an-object", "t08") @@ -80,11 +80,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(""" @@ -98,7 +102,8 @@ """) -lesson.code(""" +lesson.code( + """ one = 256 two = 256 print("256 is 256 ->", one is two) @@ -114,7 +119,10 @@ first = [] second = [] print("[] is [] ->", first is second) -""") +""", + differs=SMALL_INTS, + quiet=True, +) lesson.md(""" @@ -221,7 +229,8 @@ """) -lesson.code(""" +lesson.code( + """ from pyxray import obj low, high = obj.small_int_range() @@ -230,7 +239,10 @@ for value in [-6, -5, 0, 255, 256, 257, 1024, 1025]: shared = "shared" if obj.shares_identity(value) else "built fresh" print(f" {value:>6} {shared}") -""") +""", + differs="On 3.14 the shared range stops at 256, so 257 and 1024 come out as built fresh. The cell asks your own interpreter rather than telling you, so both answers are right.", + quiet=True, +) lesson.md(""" @@ -251,7 +263,8 @@ """) -lesson.code(""" +lesson.code( + """ source = "a = 257\\nb = 257\\n" code = compile(source, "", "exec") @@ -260,7 +273,9 @@ scope = {} exec(code, scope) print("a is b ->", scope["a"] is scope["b"], " because there is one constant, not two") -""") +""", + differs=TRAILING_NONE, +) lesson.md(""" @@ -270,7 +285,8 @@ """) -lesson.code(""" +lesson.code( + """ def fresh(text): return int(text) @@ -278,7 +294,10 @@ def fresh(text): print("int('257') twice ->", fresh("257") is fresh("257")) print("int('10') twice ->", fresh("10") is fresh("10")) print("int('99999') twice->", fresh("99999") is fresh("99999")) -""") +""", + differs=SMALL_INTS, + quiet=True, +) lesson.md(f""" diff --git a/lessons/t08-everything-is-an-object/t08.ipynb b/lessons/t08-everything-is-an-object/t08.ipynb index 3f7fc3d..d702805 100644 --- a/lessons/t08-everything-is-an-object/t08.ipynb +++ b/lessons/t08-everything-is-an-object/t08.ipynb @@ -72,7 +72,11 @@ "cell_type": "code", "execution_count": null, "id": "t08-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -98,7 +102,11 @@ "cell_type": "code", "execution_count": null, "id": "t08-07", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the shared range of small integers stops at 256 rather than 1024, so anything above 256 is a fresh object there and this prints False where the text says True." + } + }, "outputs": [], "source": [ "one = 256\n", @@ -257,7 +265,11 @@ "cell_type": "code", "execution_count": null, "id": "t08-17", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the shared range stops at 256, so 257 and 1024 come out as built fresh. The cell asks your own interpreter rather than telling you, so both answers are right." + } + }, "outputs": [], "source": [ "from pyxray import obj\n", @@ -298,7 +310,11 @@ "cell_type": "code", "execution_count": null, "id": "t08-20", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "source = \"a = 257\\nb = 257\\n\"\n", @@ -315,6 +331,14 @@ "cell_type": "markdown", "id": "t08-21", "metadata": {}, + "source": [ + "> **Version note.** On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + ] + }, + { + "cell_type": "markdown", + "id": "t08-22", + "metadata": {}, "source": [ "One 257 in the tuple and two instructions loading it, so the cache never came into it.\n", "\n", @@ -324,8 +348,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-22", - "metadata": {}, + "id": "t08-23", + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the shared range of small integers stops at 256 rather than 1024, so anything above 256 is a fresh object there and this prints False where the text says True." + } + }, "outputs": [], "source": [ "def fresh(text):\n", @@ -339,7 +367,7 @@ }, { "cell_type": "markdown", - "id": "t08-23", + "id": "t08-24", "metadata": {}, "source": [ "That last block is measuring the shelf. On 3.15 the first two say True and the third says False. On 3.14 the first says False, because 257 is past the old limit.\n", @@ -355,7 +383,7 @@ }, { "cell_type": "markdown", - "id": "t08-24", + "id": "t08-25", "metadata": {}, "source": [ "![six string literals with whether they are interned and why](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t08-everything-is-an-object/diagrams/what-gets-interned.svg)\n", @@ -370,7 +398,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-25", + "id": "t08-26", "metadata": {}, "outputs": [], "source": [ @@ -382,7 +410,7 @@ }, { "cell_type": "markdown", - "id": "t08-26", + "id": "t08-27", "metadata": {}, "source": [ "That is the same list as the table above, measured on your interpreter rather than quoted from the prose.\n", @@ -393,7 +421,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-27", + "id": "t08-28", "metadata": {}, "outputs": [], "source": [ @@ -411,7 +439,7 @@ }, { "cell_type": "markdown", - "id": "t08-28", + "id": "t08-29", "metadata": {}, "source": [ "## Counting who is holding it\n", @@ -423,7 +451,7 @@ }, { "cell_type": "markdown", - "id": "t08-29", + "id": "t08-30", "metadata": {}, "source": [ "![LOAD_FAST_BORROW adds nothing, LOAD_GLOBAL adds one, side by side](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t08-everything-is-an-object/diagrams/borrowed-or-not.svg)\n", @@ -438,7 +466,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-30", + "id": "t08-31", "metadata": {}, "outputs": [], "source": [ @@ -461,7 +489,7 @@ }, { "cell_type": "markdown", - "id": "t08-31", + "id": "t08-32", "metadata": {}, "source": [ "Both lists are held in exactly one place. The raw numbers disagree and the corrected ones do not.\n", @@ -474,7 +502,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-32", + "id": "t08-33", "metadata": {}, "outputs": [], "source": [ @@ -507,7 +535,7 @@ }, { "cell_type": "markdown", - "id": "t08-33", + "id": "t08-34", "metadata": {}, "source": [ "One, three, four, two, one. Each container that holds the object holds a reference, and dropping the container drops the reference.\n", @@ -520,7 +548,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-34", + "id": "t08-35", "metadata": {}, "outputs": [], "source": [ @@ -536,7 +564,7 @@ }, { "cell_type": "markdown", - "id": "t08-35", + "id": "t08-36", "metadata": {}, "source": [ "Two dicts and a list. The list and one of the dicts are the two containers the cell built. The other dict is the notebook's own namespace, which is holding `target` because `target` is a name at the top level, and that is the extra reference the previous cell went out of its way to avoid.\n", @@ -555,7 +583,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-36", + "id": "t08-37", "metadata": {}, "outputs": [], "source": [ @@ -580,7 +608,7 @@ }, { "cell_type": "markdown", - "id": "t08-37", + "id": "t08-38", "metadata": {}, "source": [ "## How big is it\n", @@ -592,7 +620,7 @@ }, { "cell_type": "markdown", - "id": "t08-38", + "id": "t08-39", "metadata": {}, "source": [ "![a bar chart of the size of None, 42, a one character string, an empty tuple, list and dict](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t08-everything-is-an-object/diagrams/sizes.svg)\n", @@ -604,7 +632,7 @@ }, { "cell_type": "markdown", - "id": "t08-39", + "id": "t08-40", "metadata": {}, "source": [ "![a bar chart of an empty list against lists of ten, a hundred and a thousand items](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t08-everything-is-an-object/diagrams/sizes-grow.svg)\n", @@ -617,7 +645,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t08-40", + "id": "t08-41", "metadata": {}, "outputs": [], "source": [ @@ -637,7 +665,7 @@ }, { "cell_type": "markdown", - "id": "t08-41", + "id": "t08-42", "metadata": {}, "source": [ "The two lists are the same size, because they are the same three pointers. Everything that makes the second one expensive is on the other end of those pointers, and `getsizeof` will not follow them for you. There is no function in the standard library that will, because \"how big is this really\" runs into shared objects and cycles and quickly stops having one answer.\n", diff --git a/lessons/t09-memory-appears-and-disappears/build.py b/lessons/t09-memory-appears-and-disappears/build.py index 1864531..4c706ff 100644 --- a/lessons/t09-memory-appears-and-disappears/build.py +++ b/lessons/t09-memory-appears-and-disappears/build.py @@ -17,7 +17,7 @@ notebook full of broken images. """ -from nbbuild import Lesson +from nbbuild import BANNER, Lesson from nbdiagram import Diagrams lesson = Lesson("t09-memory-appears-and-disappears", "t09") @@ -80,11 +80,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(""" @@ -280,7 +284,8 @@ def timing(): """) -lesson.code(""" +lesson.code( + """ import gc @@ -303,7 +308,9 @@ def compare(): compare() -""") +""", + varies="The number of objects the collector reports freeing counts whatever else the interpreter had waiting as well as the cycle, so it depends on the version and on what has already run. Which of the three nodes survive does not.", +) lesson.md(f""" @@ -521,7 +528,8 @@ def __init__(self): """) -lesson.code(""" +lesson.code( + """ before = heap.allocated() kept = [Plain() for _ in range(10_000)] after = heap.allocated() @@ -533,7 +541,9 @@ def __init__(self): print("blocks in use at the start ", before) print("after building 10000 ", after) print("after dropping them ", freed) -""") +""", + varies="These are counts of blocks in your own process, so the two outer numbers depend on what the interpreter has already done. The 10000 that appear and then go away again is the part to read.", +) lesson.md(""" diff --git a/lessons/t09-memory-appears-and-disappears/t09.ipynb b/lessons/t09-memory-appears-and-disappears/t09.ipynb index 47cbea5..0218c89 100644 --- a/lessons/t09-memory-appears-and-disappears/t09.ipynb +++ b/lessons/t09-memory-appears-and-disappears/t09.ipynb @@ -72,7 +72,11 @@ "cell_type": "code", "execution_count": null, "id": "t09-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -318,7 +322,11 @@ "cell_type": "code", "execution_count": null, "id": "t09-17", - "metadata": {}, + "metadata": { + "cpython_internals": { + "varies": "The number of objects the collector reports freeing counts whatever else the interpreter had waiting as well as the cycle, so it depends on the version and on what has already run. Which of the three nodes survive does not." + } + }, "outputs": [], "source": [ "import gc\n", @@ -349,6 +357,14 @@ "cell_type": "markdown", "id": "t09-18", "metadata": {}, + "source": [ + "> **Version note.** The number of objects the collector reports freeing counts whatever else the interpreter had waiting as well as the cycle, so it depends on the version and on what has already run. Which of the three nodes survive does not." + ] + }, + { + "cell_type": "markdown", + "id": "t09-19", + "metadata": {}, "source": [ "`lonely` is already gone before the collector is asked anything. The other two need it.\n", "\n", @@ -363,7 +379,7 @@ }, { "cell_type": "markdown", - "id": "t09-19", + "id": "t09-20", "metadata": {}, "source": [ "![the three steps the collector takes to tell garbage from live data](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t09-memory-appears-and-disappears/diagrams/the-subtract-trick.svg)\n", @@ -386,7 +402,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-20", + "id": "t09-21", "metadata": {}, "outputs": [], "source": [ @@ -408,7 +424,7 @@ }, { "cell_type": "markdown", - "id": "t09-21", + "id": "t09-22", "metadata": {}, "source": [ "One cycle with three members, closed into a ring. The order the names print in is the order the search finished them in rather than the direction the references run, so read it as a membership list rather than a route.\n", @@ -425,7 +441,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-22", + "id": "t09-23", "metadata": {}, "outputs": [], "source": [ @@ -448,7 +464,7 @@ }, { "cell_type": "markdown", - "id": "t09-23", + "id": "t09-24", "metadata": {}, "source": [ "Both `__del__` methods run, `gc.garbage` stays empty, and the memory comes back. If you find advice online about avoiding `__del__` because it leaks cycles, it was written before 2014.\n", @@ -464,7 +480,7 @@ }, { "cell_type": "markdown", - "id": "t09-24", + "id": "t09-25", "metadata": {}, "source": [ "![the three collector generations and how often each is examined](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t09-memory-appears-and-disappears/diagrams/generations.svg)\n", @@ -475,7 +491,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-25", + "id": "t09-26", "metadata": {}, "outputs": [], "source": [ @@ -501,7 +517,7 @@ }, { "cell_type": "markdown", - "id": "t09-26", + "id": "t09-27", "metadata": {}, "source": [ "`(2000, 10, 10)` and then 0, 1, 2, and `None`.\n", @@ -518,7 +534,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-27", + "id": "t09-28", "metadata": {}, "outputs": [], "source": [ @@ -530,7 +546,7 @@ }, { "cell_type": "markdown", - "id": "t09-28", + "id": "t09-29", "metadata": {}, "source": [ "The two tuples are the interesting pair. A tuple is a container, so it starts out tracked, but a tuple holding only untracked things can never be on a cycle either. The collector notices this the first time it looks at one and stops tracking it. That is why `(1, 2)` prints False here and would print True if you built it and asked immediately, and why `(1, [2])` stays tracked forever.\n", @@ -544,7 +560,7 @@ }, { "cell_type": "markdown", - "id": "t09-29", + "id": "t09-30", "metadata": {}, "source": [ "![blocks inside a pool inside an arena inside the operating system](https://raw.githubusercontent.com/tamnd/cpython-internals/main/lessons/t09-memory-appears-and-disappears/diagrams/the-allocator-layers.svg)\n", @@ -571,7 +587,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-30", + "id": "t09-31", "metadata": {}, "outputs": [], "source": [ @@ -588,7 +604,7 @@ }, { "cell_type": "markdown", - "id": "t09-31", + "id": "t09-32", "metadata": {}, "source": [ "An empty list is 56 bytes and takes a 64 byte block, so 8 bytes go unused, and that is the price of the whole arrangement.\n", @@ -615,8 +631,12 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-32", - "metadata": {}, + "id": "t09-33", + "metadata": { + "cpython_internals": { + "varies": "These are counts of blocks in your own process, so the two outer numbers depend on what the interpreter has already done. The 10000 that appear and then go away again is the part to read." + } + }, "outputs": [], "source": [ "before = heap.allocated()\n", @@ -634,7 +654,15 @@ }, { "cell_type": "markdown", - "id": "t09-33", + "id": "t09-34", + "metadata": {}, + "source": [ + "> **Version note.** These are counts of blocks in your own process, so the two outer numbers depend on what the interpreter has already done. The 10000 that appear and then go away again is the part to read." + ] + }, + { + "cell_type": "markdown", + "id": "t09-35", "metadata": {}, "source": [ "The middle number is about ten thousand higher and the last one is back where it started, so the blocks came back. Whether the operating system ever hears about it is a separate question, and the answer is usually no.\n", @@ -645,7 +673,7 @@ { "cell_type": "code", "execution_count": null, - "id": "t09-34", + "id": "t09-36", "metadata": {}, "outputs": [], "source": [ @@ -663,7 +691,7 @@ }, { "cell_type": "markdown", - "id": "t09-35", + "id": "t09-37", "metadata": {}, "source": [ "The same address, almost every time. The first object freed its block back to a pool, and the very next request for that size class got the same block. This is also a good reminder that `id()` is only unique among objects that are alive at the same moment, which is the footnote T08 put on it.\n", diff --git a/lessons/t10-the-napkin/build.py b/lessons/t10-the-napkin/build.py index abe0ad1..a0eb99f 100644 --- a/lessons/t10-the-napkin/build.py +++ b/lessons/t10-the-napkin/build.py @@ -13,7 +13,7 @@ them. `just lessons` checks that the committed notebook still matches this file. """ -from nbbuild import Lesson +from nbbuild import BANNER, SMALL_INTS, TRAILING_NONE, Lesson from nbdiagram import Diagrams lesson = Lesson("t10-the-napkin", "t10") @@ -74,11 +74,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(""" @@ -153,7 +157,8 @@ """) -lesson.code(""" +lesson.code( + """ import marshal @@ -183,7 +188,10 @@ def crossing(): crossing() -""") +""", + differs="On 3.14 the marshalled code object is a few bytes smaller. The number is not the point, the fact that it still runs without the source is.", + quiet=True, +) lesson.md(f""" @@ -202,7 +210,8 @@ def crossing(): """) -lesson.code(""" +lesson.code( + """ import ast import symtable import tokenize @@ -248,7 +257,10 @@ def walk_it(): walk_it() -""") +""", + differs=TRAILING_NONE, + quiet=True, +) lesson.md(""" @@ -271,7 +283,8 @@ def walk_it(): """) -lesson.code(""" +lesson.code( + """ import dis import gc @@ -319,7 +332,10 @@ def uses_a_local(): gc.collect() settle() -""") +""", + differs="On 3.14 the first answer is 10 bytes rather than 12, because the trailing None is a LOAD_CONST there. It is still bytecode, which is what the question was.", + quiet=True, +) lesson.md(f""" @@ -340,7 +356,8 @@ def uses_a_local(): """) -lesson.code(""" +lesson.code( + """ a = 257 b = 257 print("two names, same source file:", a is b) @@ -348,7 +365,10 @@ def uses_a_local(): built = int("257") again = int("257") print("built one at a time: ", built is again) -""") +""", + differs=SMALL_INTS, + quiet=True, +) lesson.md(f""" diff --git a/lessons/t10-the-napkin/t10.ipynb b/lessons/t10-the-napkin/t10.ipynb index 5984497..becf02f 100644 --- a/lessons/t10-the-napkin/t10.ipynb +++ b/lessons/t10-the-napkin/t10.ipynb @@ -70,7 +70,11 @@ "cell_type": "code", "execution_count": null, "id": "t10-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -166,7 +170,11 @@ "cell_type": "code", "execution_count": null, "id": "t10-10", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the marshalled code object is a few bytes smaller. The number is not the point, the fact that it still runs without the source is." + } + }, "outputs": [], "source": [ "import marshal\n", @@ -226,7 +234,11 @@ "cell_type": "code", "execution_count": null, "id": "t10-13", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + } + }, "outputs": [], "source": [ "import ast\n", @@ -306,7 +318,11 @@ "cell_type": "code", "execution_count": null, "id": "t10-16", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the first answer is 10 bytes rather than 12, because the trailing None is a LOAD_CONST there. It is still bytecode, which is what the question was." + } + }, "outputs": [], "source": [ "import dis\n", @@ -386,7 +402,11 @@ "cell_type": "code", "execution_count": null, "id": "t10-19", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "On 3.14 the shared range of small integers stops at 256 rather than 1024, so anything above 256 is a fresh object there and this prints False where the text says True." + } + }, "outputs": [], "source": [ "a = 257\n", diff --git a/lessons/z01-reading-c/build.py b/lessons/z01-reading-c/build.py index 9aa7845..0d31dbe 100644 --- a/lessons/z01-reading-c/build.py +++ b/lessons/z01-reading-c/build.py @@ -14,7 +14,7 @@ them. `just lessons` checks that the committed notebook still matches this file. """ -from nbbuild import Lesson +from nbbuild import BANNER, Lesson from nbdiagram import Diagrams lesson = Lesson("z01-reading-c", "z01") @@ -77,11 +77,15 @@ """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" diff --git a/lessons/z01-reading-c/z01.ipynb b/lessons/z01-reading-c/z01.ipynb index b66e447..d05c85a 100644 --- a/lessons/z01-reading-c/z01.ipynb +++ b/lessons/z01-reading-c/z01.ipynb @@ -72,7 +72,11 @@ "cell_type": "code", "execution_count": null, "id": "z01-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", diff --git a/lessons/z02-being-lost/build.py b/lessons/z02-being-lost/build.py index 6f758c8..33e0dbf 100644 --- a/lessons/z02-being-lost/build.py +++ b/lessons/z02-being-lost/build.py @@ -19,7 +19,7 @@ them. `just lessons` checks that the committed notebook still matches this file. """ -from nbbuild import Lesson +from nbbuild import BANNER, YOUR_INSTALL, Lesson from nbdiagram import Diagrams lesson = Lesson("z02-being-lost", "z02") @@ -76,15 +76,19 @@ lesson.md(""" ## Which Python is this -Most of this lesson is about a source tree rather than a running interpreter, but two of the cells read your own installation, and what they find depends on how it was built. +Most of this lesson is about a source tree rather than a running interpreter, but several of the cells read your own installation, and what they find depends on how it was built. The two cells that count files in your standard library will not match the numbers in the text, and they are not meant to. A framework install, a source build and a Colab image all ship a different set of files, and the shape of the answer is what the lesson is after. """) -lesson.code(""" +lesson.code( + """ import pyxray pyxray.show() -""") +""", + differs=BANNER, + quiet=True, +) lesson.md(f""" @@ -106,7 +110,8 @@ """) -lesson.code(""" +lesson.code( + """ import pathlib import sysconfig @@ -116,7 +121,10 @@ print("your standard library is at:", stdlib) print("files of Python:", len(sources)) print("lines of Python:", sum(len(path.read_bytes().splitlines()) for path in sources)) -""") +""", + varies=YOUR_INSTALL, + quiet=True, +) lesson.md(f""" @@ -157,7 +165,8 @@ """) -lesson.code(''' +lesson.code( + ''' MARKERS = ("generated", "do not edit", "autogenerated") @@ -172,7 +181,10 @@ def looks_generated(path): print("of your", len(sources), "files,", len(generated), "were written by a script") for path in generated[:6]: print(" ", path.relative_to(stdlib)) -''') +''', + varies=YOUR_INSTALL, + quiet=True, +) lesson.md(""" @@ -182,7 +194,8 @@ def looks_generated(path): """) -lesson.code(""" +lesson.code( + """ for path in generated: if path.name == "_opcode_metadata.py": print(path) @@ -191,7 +204,9 @@ def looks_generated(path): break else: print("not in this build, which happens on some installs") -""") +""", + differs="This file is generated from Python/bytecodes.c, so it lists the specializations your version has. On 3.14 it is a plain dict rather than a frozendict, and the list is shorter, because 3.15 added several.", +) lesson.md(f""" @@ -335,7 +350,8 @@ def where(question): """) -lesson.code(""" +lesson.code( + """ import sys builtin = set(sys.builtin_module_names) @@ -347,7 +363,9 @@ def where(question): print("Python outside, C inside: ", len(pairs)) print() print(pairs) -""") +""", + varies="How many modules are compiled into the binary is a build choice rather than a version. A framework install has far fewer than a source build, so the middle number moves a lot.", +) lesson.md(""" @@ -359,14 +377,17 @@ def where(question): """) -lesson.code(""" +lesson.code( + """ import _json import json for module in (json, _json, sys): location = getattr(module, "__file__", "no file, it is inside the binary") print(f"{module.__name__:6} {location}") -""") +""", + varies="Whether _json is a separate file or lives inside the binary is that same build choice, so this line differs between two installs of the same version.", +) lesson.md(""" diff --git a/lessons/z02-being-lost/z02.ipynb b/lessons/z02-being-lost/z02.ipynb index 7ae5a87..4e9e520 100644 --- a/lessons/z02-being-lost/z02.ipynb +++ b/lessons/z02-being-lost/z02.ipynb @@ -63,14 +63,18 @@ "source": [ "## Which Python is this\n", "\n", - "Most of this lesson is about a source tree rather than a running interpreter, but two of the cells read your own installation, and what they find depends on how it was built." + "Most of this lesson is about a source tree rather than a running interpreter, but several of the cells read your own installation, and what they find depends on how it was built. The two cells that count files in your standard library will not match the numbers in the text, and they are not meant to. A framework install, a source build and a Colab image all ship a different set of files, and the shape of the answer is what the lesson is after." ] }, { "cell_type": "code", "execution_count": null, "id": "z02-05", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This prints the interpreter you are on, so it is different for everybody." + } + }, "outputs": [], "source": [ "import pyxray\n", @@ -104,7 +108,11 @@ "cell_type": "code", "execution_count": null, "id": "z02-07", - "metadata": {}, + "metadata": { + "cpython_internals": { + "varies": "These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." + } + }, "outputs": [], "source": [ "import pathlib\n", @@ -163,7 +171,11 @@ "cell_type": "code", "execution_count": null, "id": "z02-09", - "metadata": {}, + "metadata": { + "cpython_internals": { + "varies": "These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." + } + }, "outputs": [], "source": [ "MARKERS = (\"generated\", \"do not edit\", \"autogenerated\")\n", @@ -196,7 +208,11 @@ "cell_type": "code", "execution_count": null, "id": "z02-11", - "metadata": {}, + "metadata": { + "cpython_internals": { + "differs": "This file is generated from Python/bytecodes.c, so it lists the specializations your version has. On 3.14 it is a plain dict rather than a frozendict, and the list is shorter, because 3.15 added several." + } + }, "outputs": [], "source": [ "for path in generated:\n", @@ -213,6 +229,14 @@ "cell_type": "markdown", "id": "z02-12", "metadata": {}, + "source": [ + "> **Version note.** This file is generated from Python/bytecodes.c, so it lists the specializations your version has. On 3.14 it is a plain dict rather than a frozendict, and the list is shorter, because 3.15 added several." + ] + }, + { + "cell_type": "markdown", + "id": "z02-13", + "metadata": {}, "source": [ "So the file on your laptop and `Python/generated_cases.c.h` in the source tree are two outputs of one program, reading one input: `Python/bytecodes.c`. That file is the source of truth for what every [instruction](https://github.com/tamnd/cpython-internals/blob/main/GLOSSARY.md#instruction) does, and it is the one you should read.\n", "\n", @@ -241,7 +265,7 @@ }, { "cell_type": "markdown", - "id": "z02-13", + "id": "z02-14", "metadata": {}, "source": [ "## The same instruction, written twice\n", @@ -294,7 +318,7 @@ }, { "cell_type": "markdown", - "id": "z02-14", + "id": "z02-15", "metadata": {}, "source": [ "## A map that fits on one screen\n", @@ -311,7 +335,7 @@ { "cell_type": "code", "execution_count": null, - "id": "z02-15", + "id": "z02-16", "metadata": {}, "outputs": [], "source": [ @@ -353,7 +377,7 @@ }, { "cell_type": "markdown", - "id": "z02-16", + "id": "z02-17", "metadata": {}, "source": [ "## Which half of the standard library is C\n", @@ -370,8 +394,12 @@ { "cell_type": "code", "execution_count": null, - "id": "z02-17", - "metadata": {}, + "id": "z02-18", + "metadata": { + "cpython_internals": { + "varies": "How many modules are compiled into the binary is a build choice rather than a version. A framework install has far fewer than a source build, so the middle number moves a lot." + } + }, "outputs": [], "source": [ "import sys\n", @@ -389,7 +417,15 @@ }, { "cell_type": "markdown", - "id": "z02-18", + "id": "z02-19", + "metadata": {}, + "source": [ + "> **Version note.** How many modules are compiled into the binary is a build choice rather than a version. A framework install has far fewer than a source build, so the middle number moves a lot." + ] + }, + { + "cell_type": "markdown", + "id": "z02-20", "metadata": {}, "source": [ "Forty seven pairs, which is a lot more than most people expect.\n", @@ -402,8 +438,12 @@ { "cell_type": "code", "execution_count": null, - "id": "z02-19", - "metadata": {}, + "id": "z02-21", + "metadata": { + "cpython_internals": { + "varies": "Whether _json is a separate file or lives inside the binary is that same build choice, so this line differs between two installs of the same version." + } + }, "outputs": [], "source": [ "import _json\n", @@ -416,7 +456,15 @@ }, { "cell_type": "markdown", - "id": "z02-20", + "id": "z02-22", + "metadata": {}, + "source": [ + "> **Version note.** Whether _json is a separate file or lives inside the binary is that same build choice, so this line differs between two installs of the same version." + ] + }, + { + "cell_type": "markdown", + "id": "z02-23", "metadata": {}, "source": [ "## Getting a copy of the tree\n", @@ -431,7 +479,7 @@ { "cell_type": "code", "execution_count": null, - "id": "z02-21", + "id": "z02-24", "metadata": {}, "outputs": [], "source": [ @@ -467,7 +515,7 @@ }, { "cell_type": "markdown", - "id": "z02-22", + "id": "z02-25", "metadata": {}, "source": [ "## Six questions\n", @@ -487,7 +535,7 @@ { "cell_type": "code", "execution_count": null, - "id": "z02-23", + "id": "z02-26", "metadata": {}, "outputs": [], "source": [ @@ -557,7 +605,7 @@ }, { "cell_type": "markdown", - "id": "z02-24", + "id": "z02-27", "metadata": {}, "source": [ "## When the code will not tell you why\n", @@ -582,7 +630,7 @@ { "cell_type": "code", "execution_count": null, - "id": "z02-25", + "id": "z02-28", "metadata": {}, "outputs": [], "source": [ @@ -621,7 +669,7 @@ }, { "cell_type": "markdown", - "id": "z02-26", + "id": "z02-29", "metadata": {}, "source": [ "## Where InternalDocs helps, and where it stops\n", @@ -637,7 +685,7 @@ }, { "cell_type": "markdown", - "id": "z02-27", + "id": "z02-30", "metadata": {}, "source": [ "## The boss: where did `_PyStackRef` come from\n", @@ -675,7 +723,7 @@ }, { "cell_type": "markdown", - "id": "z02-28", + "id": "z02-31", "metadata": {}, "source": [ "## How much of this you actually need\n", @@ -691,7 +739,7 @@ }, { "cell_type": "markdown", - "id": "z02-29", + "id": "z02-32", "metadata": {}, "source": [ "## Try it yourself\n", @@ -709,7 +757,7 @@ }, { "cell_type": "markdown", - "id": "z02-30", + "id": "z02-33", "metadata": {}, "source": [ "## What just happened\n", @@ -727,7 +775,7 @@ }, { "cell_type": "markdown", - "id": "z02-31", + "id": "z02-34", "metadata": {}, "source": [ "## Where this goes next\n", diff --git a/pyproject.toml b/pyproject.toml index 7a519c5..6db043e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,9 @@ ignore = [ # The glossary is the same thing without the notebook around it: definitions written as # one paragraph per line, so that a sentence is never split across two of them. "pyxray/src/pyxray/glossary.py" = ["E501"] +# The shared version notes are prose too, one sentence per line, and they are read by a +# person in a notebook rather than in the source. Same reason as the lesson builders. +"tools/nbbuild/src/nbbuild/notes.py" = ["E501"] # Every word a widget shows a reader lives in one dictionary, written one sentence per line. # Splitting a sentence across two source lines to please a line length rule would leave the # sentence looking wrapped in the source and identical on screen, which helps nobody. diff --git a/tools/nbbuild/src/nbbuild/__init__.py b/tools/nbbuild/src/nbbuild/__init__.py index f497fad..c800cdf 100644 --- a/tools/nbbuild/src/nbbuild/__init__.py +++ b/tools/nbbuild/src/nbbuild/__init__.py @@ -8,6 +8,18 @@ from __future__ import annotations from .lesson import BADGE_IMAGE, COLAB, Lesson, Malformed, repository_root +from .notes import BANNER, OFFSETS, SMALL_INTS, TRAILING_NONE, YOUR_INSTALL -__all__ = ["BADGE_IMAGE", "COLAB", "Lesson", "Malformed", "repository_root"] +__all__ = [ + "BADGE_IMAGE", + "BANNER", + "COLAB", + "OFFSETS", + "SMALL_INTS", + "TRAILING_NONE", + "YOUR_INSTALL", + "Lesson", + "Malformed", + "repository_root", +] __version__ = "0.1.0" diff --git a/tools/nbbuild/src/nbbuild/lesson.py b/tools/nbbuild/src/nbbuild/lesson.py index 008cba5..9ffcf2e 100644 --- a/tools/nbbuild/src/nbbuild/lesson.py +++ b/tools/nbbuild/src/nbbuild/lesson.py @@ -23,7 +23,7 @@ from dataclasses import dataclass, field from pathlib import Path -from nbversion.declare import KEY, NAMESPACE +from nbversion.declare import DIFFERS, NAMESPACE, VARIES from pyxray.cite import markdown as cite_markdown from pyxray.glossary import link as glossary_link @@ -140,7 +140,7 @@ def md(self, text: str) -> None: self._forbid(text) self._add("markdown", text, {}) - def code(self, text: str, *, differs: str = "", quiet: bool = False) -> None: + def code(self, text: str, *, differs: str = "", varies: str = "", quiet: bool = False) -> None: """A code cell, with no outputs and no execution count. Outputs are never committed. The only proof a cell works is CI executing it, and a @@ -154,22 +154,32 @@ def code(self, text: str, *, differs: str = "", quiet: bool = False) -> None: compare` checks it against what the two interpreters actually printed, and it comes out underneath the cell as a note the reader can see. - `quiet` turns off that second half, for the lessons where one paragraph near the top + `varies` is the other kind of note, for a cell whose output depends on the reader's + machine rather than on the version: how their interpreter was configured, how many + files their standard library has, how deep the C stack goes before it runs out. It + reads the same to a reader and is treated differently by the check, which reports it + and never fails on it, because two recordings cannot tell you whether a machine + difference is real. + + `quiet` turns off the visible cell, for the lessons where one paragraph near the top already explains a difference that then shows up in a dozen cells. Repeating it under every one of them would train the reader to skip the notes, which is the opposite of what they are for. """ + if differs and varies: + raise Malformed(f"cell {len(self.cells) + 1} is both differs and varies") extra = {"execution_count": None, "outputs": []} - if not differs: + note = differs or varies + if not note: self._add("code", text, extra) return # Checked before either cell is added, so a rejected note does not leave half of # itself behind in a lesson somebody is building interactively. - self._forbid(differs) - extra["metadata"] = {NAMESPACE: {KEY: differs}} + self._forbid(note) + extra["metadata"] = {NAMESPACE: {DIFFERS if differs else VARIES: note}} self._add("code", text, extra) if not quiet: - self.md(VERSION_NOTE.format(text=differs)) + self.md(VERSION_NOTE.format(text=note)) def document(self) -> str: """The finished notebook as the exact text that belongs on disk.""" diff --git a/tools/nbbuild/src/nbbuild/notes.py b/tools/nbbuild/src/nbbuild/notes.py new file mode 100644 index 0000000..e17f68e --- /dev/null +++ b/tools/nbbuild/src/nbbuild/notes.py @@ -0,0 +1,33 @@ +"""The version notes that more than one lesson needs to say. + +Four differences between 3.14 and 3.15 turn up over and over, in cells that have nothing +else in common. Writing the sentence once means a reader who meets it in T05 and again in +T10 gets the same words both times, and means that when 3.16 changes one of them there is +one place to fix rather than thirty. + +A note that only one lesson needs is written in that lesson's `build.py`, not here. This is +for the ones that repeat. +""" + +from __future__ import annotations + +#: Every lesson opens by printing which interpreter is about to run it. That output is +#: different for everybody, and the banner says so itself, so these are declared quietly. +BANNER = "This prints the interpreter you are on, so it is different for everybody." + +#: The one that started the whole check. 3.15 added LOAD_COMMON_CONSTANT, which carries +#: None in the instruction rather than in the code object's constant table. +TRAILING_NONE = "On 3.14 the implicit return None at the end is a LOAD_CONST and None sits in co_consts, so you get one more constant and two fewer bytes of bytecode than the text says." + +#: RESUME and GET_ITER gained an inline cache entry in 3.15, which moves everything after +#: the first instruction. Any cell that prints an offset or a byte count says this. +OFFSETS = "On 3.14 RESUME and GET_ITER have no inline cache, so every offset below is two to four lower than the numbers in the text. The shape of the listing is the same." + +#: 3.15 widened the range of integers the interpreter keeps one shared copy of, which +#: changes the answer to `257 is 257` and is the central observation of two lessons. +SMALL_INTS = "On 3.14 the shared range of small integers stops at 256 rather than 1024, so anything above 256 is a fresh object there and this prints False where the text says True." + +#: Counts taken from the installation the reader happens to have. These differ between two +#: machines running the same version, so this one goes on a cell with `varies=` rather than +#: `differs=`. A reader comparing their screen against the page still needs to be told. +YOUR_INSTALL = "These numbers describe the Python you are running rather than the language, so they will not match the text exactly. A framework install, a source build and a Colab image all count different files." diff --git a/tools/nbbuild/tests/test_nbbuild_lesson.py b/tools/nbbuild/tests/test_nbbuild_lesson.py index 88315bf..c3bdaec 100644 --- a/tools/nbbuild/tests/test_nbbuild_lesson.py +++ b/tools/nbbuild/tests/test_nbbuild_lesson.py @@ -5,7 +5,7 @@ import pytest from nbbuild import Lesson, Malformed -from nbversion.declare import KEY, NAMESPACE +from nbversion.declare import DIFFERS, NAMESPACE, VARIES @pytest.fixture @@ -55,7 +55,7 @@ def test_a_version_note_goes_in_the_cells_own_metadata(root): lesson = Lesson("t99-example", "t99", root=root) lesson.code("print(1)", differs="On 3.14 this prints nothing.") cell = json.loads(lesson.document())["cells"][0] - assert cell["metadata"] == {NAMESPACE: {KEY: "On 3.14 this prints nothing."}} + assert cell["metadata"] == {NAMESPACE: {DIFFERS: "On 3.14 this prints nothing."}} def test_a_version_note_also_comes_out_as_something_the_reader_can_see(root): @@ -73,7 +73,7 @@ def test_a_quiet_version_note_is_declared_without_a_cell_under_it(root): lesson.code("print(1)", differs="Offsets are 2 lower on 3.14.", quiet=True) cells = json.loads(lesson.document())["cells"] assert [cell["cell_type"] for cell in cells] == ["code"] - assert cells[0]["metadata"] == {NAMESPACE: {KEY: "Offsets are 2 lower on 3.14."}} + assert cells[0]["metadata"] == {NAMESPACE: {DIFFERS: "Offsets are 2 lower on 3.14."}} def test_a_version_note_goes_through_the_same_punctuation_check_as_the_prose(root): @@ -82,6 +82,38 @@ def test_a_version_note_goes_through_the_same_punctuation_check_as_the_prose(roo lesson.code("print(1)", differs="On 3.14 \u2014 nothing.") +def test_a_machine_note_goes_under_the_other_key(root): + """Same sentence to a reader, different question to the checker.""" + lesson = Lesson("t99-example", "t99", root=root) + lesson.code("print(1)", varies="How many files your standard library has.") + cells = json.loads(lesson.document())["cells"] + assert cells[0]["metadata"] == { + NAMESPACE: {VARIES: "How many files your standard library has."} + } + assert cells[1]["source"] == ["> **Version note.** How many files your standard library has."] + + +def test_a_machine_note_can_be_quiet_too(root): + lesson = Lesson("t99-example", "t99", root=root) + lesson.code("print(1)", varies="Depends on the build.", quiet=True) + cells = json.loads(lesson.document())["cells"] + assert [cell["cell_type"] for cell in cells] == ["code"] + + +def test_a_machine_note_goes_through_the_punctuation_check_as_well(root): + lesson = Lesson("t99-example", "t99", root=root) + with pytest.raises(Malformed, match="em dash"): + lesson.code("print(1)", varies="Your build \u2014 not the version.") + + +def test_a_cell_cannot_be_both_kinds_of_note(root): + """One of the two is wrong, and guessing which would put the wrong thing in metadata.""" + lesson = Lesson("t99-example", "t99", root=root) + with pytest.raises(Malformed, match="both differs and varies"): + lesson.code("print(1)", differs="version", varies="machine") + assert json.loads(lesson.document())["cells"] == [] + + def test_source_keeps_its_newlines_the_way_the_format_wants_them(root): lesson = Lesson("t99-example", "t99", root=root) lesson.code("one\ntwo") diff --git a/tools/nbversion/README.md b/tools/nbversion/README.md index 81ed5ea..bd50289 100644 --- a/tools/nbversion/README.md +++ b/tools/nbversion/README.md @@ -19,11 +19,12 @@ So the lessons are executed on both interpreters and the outputs compared. Anyth `record` runs on one interpreter and writes a small JSON file per notebook: cell id to normalised output. It is not an executed notebook, because the diff of two executed notebooks is mostly metadata. -`compare` reads two of those directories and produces one of four verdicts per cell. +`compare` reads two of those directories and produces one of five verdicts per cell. | verdict | what it means | fails | | --- | --- | --- | | `declared` | the cell differs and the notebook says so | no | +| `noted` | the cell's output depends on the machine, so there is nothing to check | no | | `undeclared` | the cell differs and nothing says so | yes | | `stale` | the cell carries a note and the two interpreters now agree | yes | | `missing` | the two runs saw different sets of cells | yes | @@ -49,6 +50,8 @@ The metadata looks like this, and survives a round trip through Jupyter because "metadata": {"cpython_internals": {"differs": "On 3.14 ..."}} ``` +`varies=` is the other keyword, for a cell whose output depends on the reader's machine rather than on the version: the flags their interpreter was built with, how many files their standard library has, how deep the C stack goes. It writes the same kind of note under a `varies` key, and it is the `noted` verdict above. Two recordings cannot check that kind of claim, because whether they agree depends on which two machines made them, and running the comparison on a CI box where both interpreters came from the same builder would call the note stale and delete something that is still true for a reader on a framework install. + ## Normalising Every substitution in `normalise.py` throws away a real difference, and the differences worth finding are exactly the ones a careless normaliser sweeps up. So a pattern gets normalised only when it varies between two runs of the *same* interpreter, which makes it noise rather than a version difference. That is addresses, absolute paths, temporary file names and durations, and nothing else. An opcode name, a size, a byte count and an offset all survive, because those are the point. diff --git a/tools/nbversion/src/nbversion/__init__.py b/tools/nbversion/src/nbversion/__init__.py index de2f418..bb8989c 100644 --- a/tools/nbversion/src/nbversion/__init__.py +++ b/tools/nbversion/src/nbversion/__init__.py @@ -11,13 +11,14 @@ """ from .compare import Finding, cells, notebooks, summary -from .declare import KEY, NAMESPACE, note, notes +from .declare import DIFFERS, NAMESPACE, VARIES, note, notes from .normalise import outputs, text from .record import Recording, run, version __all__ = [ - "KEY", + "DIFFERS", "NAMESPACE", + "VARIES", "Finding", "Recording", "cells", diff --git a/tools/nbversion/src/nbversion/cli.py b/tools/nbversion/src/nbversion/cli.py index 9ffa4b7..f7992ed 100644 --- a/tools/nbversion/src/nbversion/cli.py +++ b/tools/nbversion/src/nbversion/cli.py @@ -20,7 +20,7 @@ from nbcheck.notebook import find from .compare import notebooks, summary -from .declare import all_notes +from .declare import VARIES, all_notes from .record import DEFAULT_ROOT, load_all, run, version, write DEFAULT_ROOTS = ["lessons"] @@ -57,8 +57,8 @@ def command_compare(args) -> int: print("there are no recordings to compare", file=sys.stderr) return 2 - declared = all_notes(find(_roots(args))) - findings = notebooks(left, right, declared) + books = find(_roots(args)) + findings = notebooks(left, right, all_notes(books), all_notes(books, VARIES)) for one in findings: stream = sys.stderr if one.failed else sys.stdout print(one.line(), file=stream) @@ -68,7 +68,8 @@ def command_compare(args) -> int: if failures: print( "a cell whose output depends on the version needs a `differs=` note on it, " - "and a note whose cell no longer differs needs removing", + "a cell whose output depends on the machine needs a `varies=` note, " + "and a `differs=` note whose cell no longer differs needs removing", file=sys.stderr, ) return 1 if failures else 0 diff --git a/tools/nbversion/src/nbversion/compare.py b/tools/nbversion/src/nbversion/compare.py index 4dbb941..a29e9ef 100644 --- a/tools/nbversion/src/nbversion/compare.py +++ b/tools/nbversion/src/nbversion/compare.py @@ -17,6 +17,11 @@ #: A cell differs and the author said it would. Reported, not a failure. DECLARED = "declared" +#: A cell carries a `varies` note, which says its output depends on the machine rather than +#: on the version. Two recordings cannot check that, so this is reported and never fails, +#: whether the two runs agreed or not. +NOTED = "noted" + #: A cell differs and nothing in the notebook says so. UNDECLARED = "undeclared" @@ -60,8 +65,14 @@ def diff(first: str, second: str, *, names: tuple[str, str], context: int = 2) - return "\n".join(lines) -def cells(first: Recording, second: Recording, declared: dict[str, str]) -> list[Finding]: +def cells( + first: Recording, + second: Recording, + declared: dict[str, str], + varies: dict[str, str] | None = None, +) -> list[Finding]: """Compare one notebook's two recordings.""" + varies = varies or {} found = [] for cell in sorted(set(first.cells) | set(second.cells)): if cell not in first.cells or cell not in second.cells: @@ -75,6 +86,13 @@ def cells(first: Recording, second: Recording, declared: dict[str, str]) -> list ) ) continue + # A cell whose output depends on the machine is checked out of the comparison + # before anything is compared. Whether these two runs agreed is an accident of + # which two machines made them, so neither answer means anything. + loose = varies.get(cell, "") + if loose: + found.append(Finding(first.notebook, cell, NOTED, loose)) + continue differs = first.cells[cell] != second.cells[cell] note = declared.get(cell, "") if differs and note: @@ -108,8 +126,10 @@ def notebooks( first: dict[str, Recording], second: dict[str, Recording], declared: dict[str, dict[str, str]], + varies: dict[str, dict[str, str]] | None = None, ) -> list[Finding]: """Compare two directories of recordings.""" + varies = varies or {} found = [] for name in sorted(set(first) | set(second)): if name not in first or name not in second: @@ -118,13 +138,13 @@ def notebooks( Finding(name, "-", MISSING, f"there is no recording of it in the {side} run") ) continue - found.extend(cells(first[name], second[name], declared.get(name, {}))) + found.extend(cells(first[name], second[name], declared.get(name, {}), varies.get(name, {}))) return found def summary(findings: list[Finding]) -> str: """One line saying how it went, for the end of the output.""" - counted = {kind: 0 for kind in (DECLARED, UNDECLARED, STALE, MISSING)} + counted = {kind: 0 for kind in (DECLARED, NOTED, UNDECLARED, STALE, MISSING)} for one in findings: counted[one.kind] += 1 parts = [f"{counted[kind]} {kind}" for kind in counted if counted[kind]] diff --git a/tools/nbversion/src/nbversion/declare.py b/tools/nbversion/src/nbversion/declare.py index 6118564..b1eaf52 100644 --- a/tools/nbversion/src/nbversion/declare.py +++ b/tools/nbversion/src/nbversion/declare.py @@ -5,9 +5,16 @@ cell into another lesson and the entry does not follow. Metadata moves with the cell, survives a round trip through Jupyter, and is what nbformat is for. -`nbbuild` writes the note from the `differs=` keyword on `Lesson.code`, and also writes a -visible markdown cell underneath saying the same thing in prose, so a reader on Colab sees -the warning without opening the metadata. +`nbbuild` writes the note from the `differs=` or `varies=` keyword on `Lesson.code`, and +also writes a visible markdown cell underneath saying the same thing in prose, so a reader +on Colab sees the warning without opening the metadata. + +There are two keys because there are two kinds of note. `differs` is a claim about the +language: this cell prints one thing on 3.14 and another on 3.15, and the comparison can +check it. `varies` is a claim about the reader's machine: how their interpreter was +configured, how many files their standard library has, how deep the C stack goes. Two +recordings cannot check that one, because whether the two runs happen to agree depends on +which machine made them. So `varies` is reported and never fails. """ from __future__ import annotations @@ -20,20 +27,23 @@ #: how you collide with one of them. NAMESPACE = "cpython_internals" -#: The key inside that namespace. Its value is the sentence explaining what differs. -KEY = "differs" +#: A sentence about a difference between the two Python versions. Checkable. +DIFFERS = "differs" +#: A sentence about a difference between two machines or two builds. Not checkable. +VARIES = "varies" -def note(cell: dict) -> str: - """The version note on one cell, or the empty string if it does not have one.""" + +def note(cell: dict, key: str = DIFFERS) -> str: + """The note of one kind on one cell, or the empty string if it does not have one.""" body = cell.get("metadata", {}).get(NAMESPACE, {}) if not isinstance(body, dict): return "" - return str(body.get(KEY, "")).strip() + return str(body.get(key, "")).strip() -def notes(path: Path) -> dict[str, str]: - """Every declared cell in one notebook, keyed by cell id. +def notes(path: Path, key: str = DIFFERS) -> dict[str, str]: + """Every cell in one notebook carrying a note of that kind, keyed by cell id. Read as JSON rather than through nbformat because this is a lookup, not an execution, and going through nbformat here would mean the comparison depends on a validator @@ -42,12 +52,12 @@ def notes(path: Path) -> dict[str, str]: book = json.loads(path.read_text(encoding="utf-8")) found = {} for cell in book.get("cells", []): - text = note(cell) + text = note(cell, key) if text and cell.get("id"): found[cell["id"]] = text return found -def all_notes(paths: list[Path]) -> dict[str, dict[str, str]]: +def all_notes(paths: list[Path], key: str = DIFFERS) -> dict[str, dict[str, str]]: """The notes for several notebooks, keyed by file name then by cell id.""" - return {path.name: notes(path) for path in paths} + return {path.name: notes(path, key) for path in paths} diff --git a/tools/nbversion/tests/test_nbversion_compare.py b/tools/nbversion/tests/test_nbversion_compare.py index ed541e8..9bfbcca 100644 --- a/tools/nbversion/tests/test_nbversion_compare.py +++ b/tools/nbversion/tests/test_nbversion_compare.py @@ -1,14 +1,26 @@ -"""The four verdicts, and which of them stop the build. +"""The five verdicts, and which of them stop the build. The one that is easy to leave out is `stale`. It is tempting to treat a note as harmless once the difference goes away, and it is not: a reader who checks a note against their own interpreter, finds it wrong, and concludes the notes are decoration has been actively misled by the thing that was supposed to help them. + +`noted` is the escape hatch for a cell whose output depends on the machine rather than on +the version, which two recordings cannot decide either way. """ from __future__ import annotations -from nbversion.compare import DECLARED, MISSING, STALE, UNDECLARED, cells, notebooks, summary +from nbversion.compare import ( + DECLARED, + MISSING, + NOTED, + STALE, + UNDECLARED, + cells, + notebooks, + summary, +) from nbversion.record import Recording @@ -115,3 +127,48 @@ def test_the_summary_counts_each_kind(): right = recording({"a": "2", "b": "2", "c": "1"}, python="3.14") found = cells(left, right, {"a": "declared", "c": "stale"}) assert summary(found) == "1 declared, 1 undeclared, 1 stale" + + +def test_a_varies_note_passes_when_the_two_runs_differ(): + left, right = recording({"t01-01": "3"}), recording({"t01-01": "4"}, python="3.14") + found = cells(left, right, {}, {"t01-01": "Depends how your Python was built."}) + assert kinds(found) == [NOTED] + assert not found[0].failed + + +def test_a_varies_note_passes_when_the_two_runs_agree(): + """Which is the whole point of it. + + Whether two machines happen to agree about a machine difference says nothing, so a + `varies` note cannot go stale the way a `differs` note can. Running the comparison on a + CI box where both interpreters came from the same builder must not delete a note that + is still true for the reader on a framework install. + """ + left, right = recording({"t01-01": "3"}), recording({"t01-01": "3"}, python="3.14") + found = cells(left, right, {}, {"t01-01": "Depends how your Python was built."}) + assert kinds(found) == [NOTED] + assert not found[0].failed + + +def test_a_varies_note_wins_over_a_differs_note_on_the_same_cell(): + """The builder refuses to write both, and the comparison does not have to trust it.""" + left, right = recording({"t01-01": "3"}), recording({"t01-01": "4"}, python="3.14") + found = cells(left, right, {"t01-01": "version"}, {"t01-01": "machine"}) + assert kinds(found) == [NOTED] + assert found[0].detail == "machine" + + +def test_varies_notes_reach_the_cells_of_the_right_notebook(): + left = {"t01.ipynb": recording({"t01-01": "3"})} + right = {"t01.ipynb": recording({"t01-01": "4"}, python="3.14")} + found = notebooks(left, right, {}, {"t01.ipynb": {"t01-01": "machine"}}) + assert kinds(found) == [NOTED] + + +def test_the_summary_counts_noted_separately_from_declared(): + left = {"t01.ipynb": recording({"t01-01": "3", "t01-02": "5"})} + right = {"t01.ipynb": recording({"t01-01": "4", "t01-02": "6"}, python="3.14")} + found = notebooks( + left, right, {"t01.ipynb": {"t01-01": "version"}}, {"t01.ipynb": {"t01-02": "machine"}} + ) + assert summary(found) == "1 declared, 1 noted" diff --git a/tools/nbversion/tests/test_nbversion_declare.py b/tools/nbversion/tests/test_nbversion_declare.py index 53cbff5..01c1e45 100644 --- a/tools/nbversion/tests/test_nbversion_declare.py +++ b/tools/nbversion/tests/test_nbversion_declare.py @@ -1,10 +1,15 @@ -"""Reading the note off a cell, including all the ways a cell might not have one.""" +"""Reading the note off a cell, including all the ways a cell might not have one. + +Two keys live in the namespace, and reading one must never pick up the other. A `varies` +note read as a `differs` note would be checked against the two recordings and go stale on +whichever machine happened to run them. +""" from __future__ import annotations import json -from nbversion.declare import KEY, NAMESPACE, all_notes, note, notes +from nbversion.declare import DIFFERS, NAMESPACE, VARIES, all_notes, note, notes def cell(identifier, body=None): @@ -19,7 +24,7 @@ def notebook(tmp_path, cells, name="t01.ipynb"): def test_a_cell_with_a_note_gives_the_sentence_back(): - assert note(cell("t01-01", {KEY: "3.14 prints two lines."})) == "3.14 prints two lines." + assert note(cell("t01-01", {DIFFERS: "3.14 prints two lines."})) == "3.14 prints two lines." def test_a_cell_with_no_metadata_at_all_has_no_note(): @@ -35,7 +40,7 @@ def test_our_namespace_without_the_key_is_not_a_note(): def test_a_note_that_is_only_whitespace_is_not_a_note(): - assert note(cell("t01-01", {KEY: " "})) == "" + assert note(cell("t01-01", {DIFFERS: " "})) == "" def test_a_namespace_that_is_not_a_mapping_is_ignored_rather_than_crashing(): @@ -44,20 +49,20 @@ def test_a_namespace_that_is_not_a_mapping_is_ignored_rather_than_crashing(): def test_a_note_is_stripped(): - assert note(cell("t01-01", {KEY: " spaced "})) == "spaced" + assert note(cell("t01-01", {DIFFERS: " spaced "})) == "spaced" def test_reading_a_notebook_gives_only_the_cells_that_carry_a_note(tmp_path): path = notebook( tmp_path, - [cell("t01-01"), cell("t01-02", {KEY: "differs"}), cell("t01-03")], + [cell("t01-01"), cell("t01-02", {DIFFERS: "differs"}), cell("t01-03")], ) assert notes(path) == {"t01-02": "differs"} def test_a_cell_with_a_note_and_no_id_is_skipped(tmp_path): """There is nothing to key it on, and every cell we generate has one.""" - path = notebook(tmp_path, [{"metadata": {NAMESPACE: {KEY: "differs"}}}]) + path = notebook(tmp_path, [{"metadata": {NAMESPACE: {DIFFERS: "differs"}}}]) assert notes(path) == {} @@ -66,6 +71,26 @@ def test_a_notebook_with_no_notes_reads_as_an_empty_mapping(tmp_path): def test_several_notebooks_are_keyed_by_file_name(tmp_path): - first = notebook(tmp_path, [cell("t01-01", {KEY: "a"})], name="t01.ipynb") + first = notebook(tmp_path, [cell("t01-01", {DIFFERS: "a"})], name="t01.ipynb") second = notebook(tmp_path, [cell("t02-01")], name="t02.ipynb") assert all_notes([first, second]) == {"t01.ipynb": {"t01-01": "a"}, "t02.ipynb": {}} + + +def test_the_two_kinds_of_note_do_not_read_as_each_other(): + machine = cell("t01-01", {VARIES: "Depends how your Python was built."}) + assert note(machine) == "" + assert note(machine, VARIES) == "Depends how your Python was built." + + +def test_reading_a_notebook_for_varies_skips_the_differs_cells(tmp_path): + path = notebook( + tmp_path, + [cell("t01-01", {DIFFERS: "version"}), cell("t01-02", {VARIES: "machine"})], + ) + assert notes(path) == {"t01-01": "version"} + assert notes(path, VARIES) == {"t01-02": "machine"} + + +def test_all_notes_takes_the_kind_too(tmp_path): + path = notebook(tmp_path, [cell("t01-01", {VARIES: "machine"})], name="t01.ipynb") + assert all_notes([path], VARIES) == {"t01.ipynb": {"t01-01": "machine"}} diff --git a/tools/nbversion/tests/version_fixtures.py b/tools/nbversion/tests/version_fixtures.py index a00c951..6886136 100644 --- a/tools/nbversion/tests/version_fixtures.py +++ b/tools/nbversion/tests/version_fixtures.py @@ -11,15 +11,21 @@ import json from pathlib import Path -from nbversion.declare import KEY, NAMESPACE +from nbversion.declare import DIFFERS, NAMESPACE, VARIES from nbversion.record import Recording from nbversion.record import write as write_recording _ids = itertools.count() -def code(source: str, *, differs: str = "", identifier: str | None = None) -> dict: - metadata = {NAMESPACE: {KEY: differs}} if differs else {} +def code( + source: str, *, differs: str = "", varies: str = "", identifier: str | None = None +) -> dict: + metadata = {} + if differs: + metadata = {NAMESPACE: {DIFFERS: differs}} + elif varies: + metadata = {NAMESPACE: {VARIES: varies}} return { "cell_type": "code", "id": identifier or f"cell-{next(_ids)}",