diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b60246..ebddcdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,9 @@ jobs: # below, which already treats blueprints/ as one of its roots. - run: uv run pytest tools/bpcheck/tests - run: uv run bpcheck lint + # bpc has tests that need a CPython checkout and tests that do not. The ones that do + # skip here and run in the citations job, which is the one with the tree. + - run: uv run pytest tools/bpc/tests animations: runs-on: ubuntu-latest @@ -157,3 +160,9 @@ jobs: - run: uv sync --all-packages - run: uv run pytest tools/refcheck/tests - run: uv run refcheck verify + # The generated sections of a blueprint are read out of Parser/Python.asdl, so this + # is the job with the tree to read them from. It fails if the committed blueprint no + # longer matches what the grammar says, which is what happens when the pin moves and + # nobody reran `just build-blueprints`. + - run: uv run bpc check + - run: uv run pytest tools/bpc/tests diff --git a/README.md b/README.md index f70c6fd..9afc4cf 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Pinned to `v3.15.0rc1` today and moving to `v3.15.0` when it ships on 1 October | `nbbuild` | Lessons are written as Python and generated into notebooks, because nobody should have to edit a `.ipynb` by hand or review a diff of one. The generated file is committed as well, and CI fails if it stops matching the code that produced it | [tools/nbbuild](tools/nbbuild) | | `nbdiagram` | Every picture in a lesson is an Excalidraw scene drawn from Python, written out as an editable `.excalidraw` and as the `.svg` GitHub and Colab display. Colours, type and spacing come from one shared theme, so the diagrams, the charts and the animations look like one project | [tools/nbdiagram](tools/nbdiagram) | | `bpcheck` | The shape a blueprint has to have before somebody can implement from it: the nine sections in order, the header block, the invariant numbering, and no fact deferred to a lesson | [tools/bpcheck](tools/bpcheck) | +| `bpc` | The blueprint compiler. Where upstream ships the material in a form a program can read, the specification is generated from it rather than typed. It reads `Parser/Python.asdl` with CPython's own parser and writes the three sections of BP-AST that list all 113 node kinds, each one citing the line it is declared on | [tools/bpc](tools/bpc) | | `xraymanim` | The animations, and the fifteen shapes they are allowed to be made of. Each one is planned as a storyboard that is checked in milliseconds, so a mistake is caught before anybody pays for a render | [xraymanim](xraymanim) | | `xraywidgets` | The parts of a lesson you can click: a disassembler that shows what `dis` hides, a pipeline explorer with six panes from source to code object, and a prediction gate that asks before it tells. Each one renders twice from one piece of code: plain HTML with nothing installed, and the same picture with working buttons when anywidget is there | [xraywidgets](xraywidgets) | @@ -113,15 +114,19 @@ Every one has the same nine sections in the same order, so a reader who has read |---|---|---| | BP-MAP | [The shape of the whole interpreter](blueprints/BP-MAP.md) | The runtime, the interpreter, the thread state and the frame, what contains what, and which source file belongs to which blueprint so that two of them cannot claim the same code | | BP-PIPELINE | [Source text to a running frame](blueprints/BP-PIPELINE.md) | The eight artifacts and the seven transitions between them, the arena that holds the middle five, the three depth limits and the three different exceptions they raise, and the exact point where compile time ends | +| BP-AST | [The node types and their fields](blueprints/BP-AST.md) | Every one of the 19 types, the 113 node kinds and the 198 fields, what each field holds and what happens when you leave it out, the arena the whole tree lives in, and the validation pass that rejects trees the grammar allows | The pseudocode is one dialect across all of them, defined in [blueprints/NOTATION.md](blueprints/NOTATION.md), with explicit pointers, explicit allocation, explicit refcount operations and no exceptions. Their citations are resolved against the pinned tree along with everything else, and `bpcheck` holds the structure up. +Sections 1, 2 and 5 of BP-AST are not typed by anybody. They are compiled out of `Parser/Python.asdl` by [bpc](tools/bpc), using CPython's own ASDL parser, so the table of node kinds is right by construction and every row cites the line it came from. The prose lives in [blueprints/sources/BP-AST.md](blueprints/sources/BP-AST.md) and the finished document is committed next to the hand written ones, with markers showing where the generated parts start and stop. + ## What is planned to be here ``` book/ the prose site, one directory per part lessons/ the notebooks, the source of truth for every runnable cell -blueprints/ the normative specification, mechanical sections generated +blueprints/ the normative specification, with sources/ holding the prose half + of the ones whose mechanical sections are generated anim/ manim scenes, built from one shared mobject library apps/ the Gradio playgrounds pyxray/ the instrumentation toolkit every lesson imports diff --git a/blueprints/BP-AST.md b/blueprints/BP-AST.md new file mode 100644 index 0000000..faa86fe --- /dev/null +++ b/blueprints/BP-AST.md @@ -0,0 +1,875 @@ +# BP-AST: the abstract syntax tree + +**Covers:** `Parser/Python.asdl`, `Python/Python-ast.c`, `Python/ast.c`, `Include/internal/pycore_ast.h` and `Include/internal/pycore_asdl.h`, at the pinned tag +**Lesson:** T03, tokens become a tree +**Status:** partial +**Compatibility tier:** B + +## 1. Purpose and scope + +This blueprint specifies the shape of the tree the parser produces and the compiler consumes. It names every node kind, every field of every node kind, the order those fields are in, whether each one is required, optional or a sequence, and which node kinds carry source locations. + +In scope: the node vocabulary, the C representation of it, how a node is allocated and how long it lives, what a Python program can see of all this through the `ast` module, and the structural checks CPython applies to a tree that was built by hand rather than parsed. + +Out of scope: how the tree gets built. The grammar and the parser that matches it are `BP-PARSER`, and this blueprint says nothing about which source text produces which node. What happens to the tree afterwards is `BP-SYMTABLE` and `BP-CODEGEN`. The order the stages run in is `BP-PIPELINE`. + +The table at the end of this section, the whole of section 2, the whole of section 5 and the whole of section 8 are generated from `Parser/Python.asdl` by `bpc`, the same file CPython generates its own node structures, its C constructors and its Python classes from. They are not typed by anybody and they are not proofread by anybody. Everything else in this document is written by hand, including the paragraphs above, because where this subsystem stops and `BP-PARSER` starts is not in the grammar and never will be. A generated table cannot be one field out of date, which is the way a hand written one goes wrong: correct on the day it is written, wrong the first time upstream adds a field, and nobody finds out because a reader who trusted the table has no reason to check it. + +The generator is `Parser/asdl_c.py:1-2@v3.15.0rc1`, run from the `regen-ast` rule at `Makefile.pre.in:2056-2072@v3.15.0rc1`, which writes three files: the node structures, the per interpreter state that holds the Python classes, and the C source that builds both. Everything this blueprint describes is downstream of one 154 line grammar file. + + +The grammar declares 19 types, 12 of them a choice between constructors and 7 of them a single fixed shape. Between them they describe 113 concrete node kinds with 198 fields, and 8 of the types carry source location attributes on every node. + +The module header is at `Parser/Python.asdl:4@v3.15.0rc1#Python`. + +| # | Type | Kind | Constructors | Fields | Attributes | Declared at | +|---|---|---|---|---|---|---| +| 1 | `mod` | sum | 4 | 6 | 0 | `Parser/Python.asdl:6@v3.15.0rc1#mod` | +| 2 | `stmt` | sum | 28 | 80 | 4 | `Parser/Python.asdl:11@v3.15.0rc1#stmt` | +| 3 | `expr` | sum | 29 | 63 | 4 | `Parser/Python.asdl:60@v3.15.0rc1#expr` | +| 4 | `expr_context` | sum | 3 | 0 | 0 | `Parser/Python.asdl:100@v3.15.0rc1#expr_context` | +| 5 | `boolop` | sum | 2 | 0 | 0 | `Parser/Python.asdl:102@v3.15.0rc1#boolop` | +| 6 | `operator` | sum | 13 | 0 | 0 | `Parser/Python.asdl:104@v3.15.0rc1#operator` | +| 7 | `unaryop` | sum | 4 | 0 | 0 | `Parser/Python.asdl:107@v3.15.0rc1#unaryop` | +| 8 | `cmpop` | sum | 10 | 0 | 0 | `Parser/Python.asdl:109@v3.15.0rc1#cmpop` | +| 9 | `comprehension` | product | | 4 | 0 | `Parser/Python.asdl:111@v3.15.0rc1#comprehension` | +| 10 | `excepthandler` | sum | 1 | 3 | 4 | `Parser/Python.asdl:113@v3.15.0rc1#excepthandler` | +| 11 | `arguments` | product | | 7 | 0 | `Parser/Python.asdl:116@v3.15.0rc1#arguments` | +| 12 | `arg` | product | | 3 | 4 | `Parser/Python.asdl:119@v3.15.0rc1#arg` | +| 13 | `keyword` | product | | 2 | 4 | `Parser/Python.asdl:123@v3.15.0rc1#keyword` | +| 14 | `alias` | product | | 2 | 4 | `Parser/Python.asdl:127@v3.15.0rc1#alias` | +| 15 | `withitem` | product | | 2 | 0 | `Parser/Python.asdl:130@v3.15.0rc1#withitem` | +| 16 | `match_case` | product | | 3 | 0 | `Parser/Python.asdl:132@v3.15.0rc1#match_case` | +| 17 | `pattern` | sum | 8 | 14 | 4 | `Parser/Python.asdl:134@v3.15.0rc1#pattern` | +| 18 | `type_ignore` | sum | 1 | 2 | 0 | `Parser/Python.asdl:148@v3.15.0rc1#type_ignore` | +| 19 | `type_param` | sum | 3 | 7 | 4 | `Parser/Python.asdl:150@v3.15.0rc1#type_param` | + + +## 2. Data structures + +Every node kind in the grammar becomes one C struct, one Python class and one constructor function. This section lists what the grammar says. How that becomes C is here, and the tables below are the specification of what a port has to build. + +### 2.0.1 The C shape of a node + +A sum type becomes a struct holding a `kind` enum and a union of one anonymous struct per constructor, so `struct _stmt` at `Include/internal/pycore_ast.h:196-207@v3.15.0rc1#_stmt` has a `kind` of `FunctionDef_kind` and a `v.FunctionDef` holding that constructor's seven fields. A product type has no `kind` and no union, because there is nothing to switch on. + +The attributes listed for a type sit outside the union, once, rather than being repeated in each arm. That is what makes `node->lineno` readable without knowing which kind of statement it is, and it is the reason attributes and fields are separate concepts rather than a naming convention. + +A field whose type is one of the four ASDL built ins becomes a `PyObject *` for `identifier`, `string` and `constant`, and a plain `int` for `int`. A field whose type is another node type becomes a pointer to that node's struct. An optional field is the same pointer and may be `NULL`. There is no separate tag saying whether an optional field is present. + +### 2.0.2 Sequences + +A sequence field is a pointer to an `asdl_seq`, which is a length and an array of pointers laid out inline after it. The header is two words, at `Include/internal/pycore_asdl.h:24-26@v3.15.0rc1#_ASDL_SEQ_HEAD`, and one typed variant is generated per element type by the macro at `Include/internal/pycore_asdl.h:52-74@v3.15.0rc1#GENERATE_ASDL_SEQ_CONSTRUCTOR`. + +The length and the elements are read through macros rather than directly, and `asdl_seq_LEN` at `Include/internal/pycore_asdl.h:83@v3.15.0rc1#asdl_seq_LEN` reports zero for a `NULL` sequence. A port that represents a sequence as a growable array gets this for free. A port that distinguishes an absent list from an empty one has invented a state CPython does not have. + +Sequences are allocated at their final length and never grown. The parser knows how many children it matched before it builds the node, so there is no append path and no capacity field. + +### 2.0.3 Lifetime + +Every node and every sequence is allocated from the compile time arena, through `_PyArena_Malloc` at `Include/internal/pycore_pyarena.h:56@v3.15.0rc1#_PyArena_Malloc`, and nothing is freed individually. The whole tree goes away in one call when the arena is released, which happens once a code object exists. + +This is why no node has a destructor and why nothing in the tree is reference counted except the `PyObject *` fields, which the arena holds a reference to on the tree's behalf. A port that allocates nodes individually has to answer a question CPython never asks: who owns a subtree that was built and then discarded when a parser alternative failed. The arena's answer is that nobody does and it does not matter. + + +### 2.1 `mod` + +A choice between 4 constructors, declared at `Parser/Python.asdl:6@v3.15.0rc1#mod`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `Module` | 1 | `body` | `stmt` | sequence | `Parser/Python.asdl:6@v3.15.0rc1#Module` | +| `Module` | 2 | `type_ignores` | `type_ignore` | sequence | | +| `Interactive` | 1 | `body` | `stmt` | sequence | `Parser/Python.asdl:7@v3.15.0rc1#Interactive` | +| `Expression` | 1 | `body` | `expr` | required | `Parser/Python.asdl:8@v3.15.0rc1#Expression` | +| `FunctionType` | 1 | `argtypes` | `expr` | sequence | `Parser/Python.asdl:9@v3.15.0rc1#FunctionType` | +| `FunctionType` | 2 | `returns` | `expr` | required | | + +### 2.2 `stmt` + +A choice between 28 constructors, declared at `Parser/Python.asdl:11@v3.15.0rc1#stmt`. + +Every `stmt` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | optional | +| `end_col_offset` | `int` | optional | + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `FunctionDef` | 1 | `name` | `identifier` | required | `Parser/Python.asdl:11@v3.15.0rc1#FunctionDef` | +| `FunctionDef` | 2 | `args` | `arguments` | required | | +| `FunctionDef` | 3 | `body` | `stmt` | sequence | | +| `FunctionDef` | 4 | `decorator_list` | `expr` | sequence | | +| `FunctionDef` | 5 | `returns` | `expr` | optional | | +| `FunctionDef` | 6 | `type_comment` | `string` | optional | | +| `FunctionDef` | 7 | `type_params` | `type_param` | sequence | | +| `AsyncFunctionDef` | 1 | `name` | `identifier` | required | `Parser/Python.asdl:14@v3.15.0rc1#AsyncFunctionDef` | +| `AsyncFunctionDef` | 2 | `args` | `arguments` | required | | +| `AsyncFunctionDef` | 3 | `body` | `stmt` | sequence | | +| `AsyncFunctionDef` | 4 | `decorator_list` | `expr` | sequence | | +| `AsyncFunctionDef` | 5 | `returns` | `expr` | optional | | +| `AsyncFunctionDef` | 6 | `type_comment` | `string` | optional | | +| `AsyncFunctionDef` | 7 | `type_params` | `type_param` | sequence | | +| `ClassDef` | 1 | `name` | `identifier` | required | `Parser/Python.asdl:18@v3.15.0rc1#ClassDef` | +| `ClassDef` | 2 | `bases` | `expr` | sequence | | +| `ClassDef` | 3 | `keywords` | `keyword` | sequence | | +| `ClassDef` | 4 | `body` | `stmt` | sequence | | +| `ClassDef` | 5 | `decorator_list` | `expr` | sequence | | +| `ClassDef` | 6 | `type_params` | `type_param` | sequence | | +| `Return` | 1 | `value` | `expr` | optional | `Parser/Python.asdl:24@v3.15.0rc1#Return` | +| `Delete` | 1 | `targets` | `expr` | sequence | `Parser/Python.asdl:26@v3.15.0rc1#Delete` | +| `Assign` | 1 | `targets` | `expr` | sequence | `Parser/Python.asdl:27@v3.15.0rc1#Assign` | +| `Assign` | 2 | `value` | `expr` | required | | +| `Assign` | 3 | `type_comment` | `string` | optional | | +| `TypeAlias` | 1 | `name` | `expr` | required | `Parser/Python.asdl:28@v3.15.0rc1#TypeAlias` | +| `TypeAlias` | 2 | `type_params` | `type_param` | sequence | | +| `TypeAlias` | 3 | `value` | `expr` | required | | +| `AugAssign` | 1 | `target` | `expr` | required | `Parser/Python.asdl:29@v3.15.0rc1#AugAssign` | +| `AugAssign` | 2 | `op` | `operator` | required | | +| `AugAssign` | 3 | `value` | `expr` | required | | +| `AnnAssign` | 1 | `target` | `expr` | required | `Parser/Python.asdl:31@v3.15.0rc1#AnnAssign` | +| `AnnAssign` | 2 | `annotation` | `expr` | required | | +| `AnnAssign` | 3 | `value` | `expr` | optional | | +| `AnnAssign` | 4 | `simple` | `int` | required | | +| `For` | 1 | `target` | `expr` | required | `Parser/Python.asdl:34@v3.15.0rc1#For` | +| `For` | 2 | `iter` | `expr` | required | | +| `For` | 3 | `body` | `stmt` | sequence | | +| `For` | 4 | `orelse` | `stmt` | sequence | | +| `For` | 5 | `type_comment` | `string` | optional | | +| `AsyncFor` | 1 | `target` | `expr` | required | `Parser/Python.asdl:35@v3.15.0rc1#AsyncFor` | +| `AsyncFor` | 2 | `iter` | `expr` | required | | +| `AsyncFor` | 3 | `body` | `stmt` | sequence | | +| `AsyncFor` | 4 | `orelse` | `stmt` | sequence | | +| `AsyncFor` | 5 | `type_comment` | `string` | optional | | +| `While` | 1 | `test` | `expr` | required | `Parser/Python.asdl:36@v3.15.0rc1#While` | +| `While` | 2 | `body` | `stmt` | sequence | | +| `While` | 3 | `orelse` | `stmt` | sequence | | +| `If` | 1 | `test` | `expr` | required | `Parser/Python.asdl:37@v3.15.0rc1#If` | +| `If` | 2 | `body` | `stmt` | sequence | | +| `If` | 3 | `orelse` | `stmt` | sequence | | +| `With` | 1 | `items` | `withitem` | sequence | `Parser/Python.asdl:38@v3.15.0rc1#With` | +| `With` | 2 | `body` | `stmt` | sequence | | +| `With` | 3 | `type_comment` | `string` | optional | | +| `AsyncWith` | 1 | `items` | `withitem` | sequence | `Parser/Python.asdl:39@v3.15.0rc1#AsyncWith` | +| `AsyncWith` | 2 | `body` | `stmt` | sequence | | +| `AsyncWith` | 3 | `type_comment` | `string` | optional | | +| `Match` | 1 | `subject` | `expr` | required | `Parser/Python.asdl:41@v3.15.0rc1#Match` | +| `Match` | 2 | `cases` | `match_case` | sequence | | +| `Raise` | 1 | `exc` | `expr` | optional | `Parser/Python.asdl:43@v3.15.0rc1#Raise` | +| `Raise` | 2 | `cause` | `expr` | optional | | +| `Try` | 1 | `body` | `stmt` | sequence | `Parser/Python.asdl:44@v3.15.0rc1#Try` | +| `Try` | 2 | `handlers` | `excepthandler` | sequence | | +| `Try` | 3 | `orelse` | `stmt` | sequence | | +| `Try` | 4 | `finalbody` | `stmt` | sequence | | +| `TryStar` | 1 | `body` | `stmt` | sequence | `Parser/Python.asdl:45@v3.15.0rc1#TryStar` | +| `TryStar` | 2 | `handlers` | `excepthandler` | sequence | | +| `TryStar` | 3 | `orelse` | `stmt` | sequence | | +| `TryStar` | 4 | `finalbody` | `stmt` | sequence | | +| `Assert` | 1 | `test` | `expr` | required | `Parser/Python.asdl:46@v3.15.0rc1#Assert` | +| `Assert` | 2 | `msg` | `expr` | optional | | +| `Import` | 1 | `names` | `alias` | sequence | `Parser/Python.asdl:48@v3.15.0rc1#Import` | +| `Import` | 2 | `is_lazy` | `int` | optional | | +| `ImportFrom` | 1 | `module` | `identifier` | optional | `Parser/Python.asdl:49@v3.15.0rc1#ImportFrom` | +| `ImportFrom` | 2 | `names` | `alias` | sequence | | +| `ImportFrom` | 3 | `level` | `int` | optional | | +| `ImportFrom` | 4 | `is_lazy` | `int` | optional | | +| `Global` | 1 | `names` | `identifier` | sequence | `Parser/Python.asdl:51@v3.15.0rc1#Global` | +| `Nonlocal` | 1 | `names` | `identifier` | sequence | `Parser/Python.asdl:52@v3.15.0rc1#Nonlocal` | +| `Expr` | 1 | `value` | `expr` | required | `Parser/Python.asdl:53@v3.15.0rc1#Expr` | +| `Pass` | | no fields | | | `Parser/Python.asdl:54@v3.15.0rc1#Pass` | +| `Break` | | no fields | | | `Parser/Python.asdl:54@v3.15.0rc1#Break` | +| `Continue` | | no fields | | | `Parser/Python.asdl:54@v3.15.0rc1#Continue` | + +### 2.3 `expr` + +A choice between 29 constructors, declared at `Parser/Python.asdl:60@v3.15.0rc1#expr`. + +Every `expr` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | optional | +| `end_col_offset` | `int` | optional | + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `BoolOp` | 1 | `op` | `boolop` | required | `Parser/Python.asdl:60@v3.15.0rc1#BoolOp` | +| `BoolOp` | 2 | `values` | `expr` | sequence | | +| `NamedExpr` | 1 | `target` | `expr` | required | `Parser/Python.asdl:61@v3.15.0rc1#NamedExpr` | +| `NamedExpr` | 2 | `value` | `expr` | required | | +| `BinOp` | 1 | `left` | `expr` | required | `Parser/Python.asdl:62@v3.15.0rc1#BinOp` | +| `BinOp` | 2 | `op` | `operator` | required | | +| `BinOp` | 3 | `right` | `expr` | required | | +| `UnaryOp` | 1 | `op` | `unaryop` | required | `Parser/Python.asdl:63@v3.15.0rc1#UnaryOp` | +| `UnaryOp` | 2 | `operand` | `expr` | required | | +| `Lambda` | 1 | `args` | `arguments` | required | `Parser/Python.asdl:64@v3.15.0rc1#Lambda` | +| `Lambda` | 2 | `body` | `expr` | required | | +| `IfExp` | 1 | `test` | `expr` | required | `Parser/Python.asdl:65@v3.15.0rc1#IfExp` | +| `IfExp` | 2 | `body` | `expr` | required | | +| `IfExp` | 3 | `orelse` | `expr` | required | | +| `Dict` | 1 | `keys` | `expr` | sequence of optional | `Parser/Python.asdl:66@v3.15.0rc1#Dict` | +| `Dict` | 2 | `values` | `expr` | sequence | | +| `Set` | 1 | `elts` | `expr` | sequence | `Parser/Python.asdl:67@v3.15.0rc1#Set` | +| `ListComp` | 1 | `elt` | `expr` | required | `Parser/Python.asdl:68@v3.15.0rc1#ListComp` | +| `ListComp` | 2 | `generators` | `comprehension` | sequence | | +| `SetComp` | 1 | `elt` | `expr` | required | `Parser/Python.asdl:69@v3.15.0rc1#SetComp` | +| `SetComp` | 2 | `generators` | `comprehension` | sequence | | +| `DictComp` | 1 | `key` | `expr` | required | `Parser/Python.asdl:70@v3.15.0rc1#DictComp` | +| `DictComp` | 2 | `value` | `expr` | optional | | +| `DictComp` | 3 | `generators` | `comprehension` | sequence | | +| `GeneratorExp` | 1 | `elt` | `expr` | required | `Parser/Python.asdl:71@v3.15.0rc1#GeneratorExp` | +| `GeneratorExp` | 2 | `generators` | `comprehension` | sequence | | +| `Await` | 1 | `value` | `expr` | required | `Parser/Python.asdl:73@v3.15.0rc1#Await` | +| `Yield` | 1 | `value` | `expr` | optional | `Parser/Python.asdl:74@v3.15.0rc1#Yield` | +| `YieldFrom` | 1 | `value` | `expr` | required | `Parser/Python.asdl:75@v3.15.0rc1#YieldFrom` | +| `Compare` | 1 | `left` | `expr` | required | `Parser/Python.asdl:78@v3.15.0rc1#Compare` | +| `Compare` | 2 | `ops` | `cmpop` | sequence | | +| `Compare` | 3 | `comparators` | `expr` | sequence | | +| `Call` | 1 | `func` | `expr` | required | `Parser/Python.asdl:79@v3.15.0rc1#Call` | +| `Call` | 2 | `args` | `expr` | sequence | | +| `Call` | 3 | `keywords` | `keyword` | sequence | | +| `FormattedValue` | 1 | `value` | `expr` | required | `Parser/Python.asdl:80@v3.15.0rc1#FormattedValue` | +| `FormattedValue` | 2 | `conversion` | `int` | required | | +| `FormattedValue` | 3 | `format_spec` | `expr` | optional | | +| `Interpolation` | 1 | `value` | `expr` | required | `Parser/Python.asdl:81@v3.15.0rc1#Interpolation` | +| `Interpolation` | 2 | `str` | `constant` | required | | +| `Interpolation` | 3 | `conversion` | `int` | required | | +| `Interpolation` | 4 | `format_spec` | `expr` | optional | | +| `JoinedStr` | 1 | `values` | `expr` | sequence | `Parser/Python.asdl:82@v3.15.0rc1#JoinedStr` | +| `TemplateStr` | 1 | `values` | `expr` | sequence | `Parser/Python.asdl:83@v3.15.0rc1#TemplateStr` | +| `Constant` | 1 | `value` | `constant` | required | `Parser/Python.asdl:84@v3.15.0rc1#Constant` | +| `Constant` | 2 | `kind` | `string` | optional | | +| `Attribute` | 1 | `value` | `expr` | required | `Parser/Python.asdl:87@v3.15.0rc1#Attribute` | +| `Attribute` | 2 | `attr` | `identifier` | required | | +| `Attribute` | 3 | `ctx` | `expr_context` | required | | +| `Subscript` | 1 | `value` | `expr` | required | `Parser/Python.asdl:88@v3.15.0rc1#Subscript` | +| `Subscript` | 2 | `slice` | `expr` | required | | +| `Subscript` | 3 | `ctx` | `expr_context` | required | | +| `Starred` | 1 | `value` | `expr` | required | `Parser/Python.asdl:89@v3.15.0rc1#Starred` | +| `Starred` | 2 | `ctx` | `expr_context` | required | | +| `Name` | 1 | `id` | `identifier` | required | `Parser/Python.asdl:90@v3.15.0rc1#Name` | +| `Name` | 2 | `ctx` | `expr_context` | required | | +| `List` | 1 | `elts` | `expr` | sequence | `Parser/Python.asdl:91@v3.15.0rc1#List` | +| `List` | 2 | `ctx` | `expr_context` | required | | +| `Tuple` | 1 | `elts` | `expr` | sequence | `Parser/Python.asdl:92@v3.15.0rc1#Tuple` | +| `Tuple` | 2 | `ctx` | `expr_context` | required | | +| `Slice` | 1 | `lower` | `expr` | optional | `Parser/Python.asdl:95@v3.15.0rc1#Slice` | +| `Slice` | 2 | `upper` | `expr` | optional | | +| `Slice` | 3 | `step` | `expr` | optional | | + +### 2.4 `expr_context` + +A choice between 3 constructors, declared at `Parser/Python.asdl:100@v3.15.0rc1#expr_context`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `Load` | | no fields | | | `Parser/Python.asdl:100@v3.15.0rc1#Load` | +| `Store` | | no fields | | | `Parser/Python.asdl:100@v3.15.0rc1#Store` | +| `Del` | | no fields | | | `Parser/Python.asdl:100@v3.15.0rc1#Del` | + +### 2.5 `boolop` + +A choice between 2 constructors, declared at `Parser/Python.asdl:102@v3.15.0rc1#boolop`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `And` | | no fields | | | `Parser/Python.asdl:102@v3.15.0rc1#And` | +| `Or` | | no fields | | | `Parser/Python.asdl:102@v3.15.0rc1#Or` | + +### 2.6 `operator` + +A choice between 13 constructors, declared at `Parser/Python.asdl:104@v3.15.0rc1#operator`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `Add` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#Add` | +| `Sub` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#Sub` | +| `Mult` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#Mult` | +| `MatMult` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#MatMult` | +| `Div` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#Div` | +| `Mod` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#Mod` | +| `Pow` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#Pow` | +| `LShift` | | no fields | | | `Parser/Python.asdl:104@v3.15.0rc1#LShift` | +| `RShift` | | no fields | | | `Parser/Python.asdl:105@v3.15.0rc1#RShift` | +| `BitOr` | | no fields | | | `Parser/Python.asdl:105@v3.15.0rc1#BitOr` | +| `BitXor` | | no fields | | | `Parser/Python.asdl:105@v3.15.0rc1#BitXor` | +| `BitAnd` | | no fields | | | `Parser/Python.asdl:105@v3.15.0rc1#BitAnd` | +| `FloorDiv` | | no fields | | | `Parser/Python.asdl:105@v3.15.0rc1#FloorDiv` | + +### 2.7 `unaryop` + +A choice between 4 constructors, declared at `Parser/Python.asdl:107@v3.15.0rc1#unaryop`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `Invert` | | no fields | | | `Parser/Python.asdl:107@v3.15.0rc1#Invert` | +| `Not` | | no fields | | | `Parser/Python.asdl:107@v3.15.0rc1#Not` | +| `UAdd` | | no fields | | | `Parser/Python.asdl:107@v3.15.0rc1#UAdd` | +| `USub` | | no fields | | | `Parser/Python.asdl:107@v3.15.0rc1#USub` | + +### 2.8 `cmpop` + +A choice between 10 constructors, declared at `Parser/Python.asdl:109@v3.15.0rc1#cmpop`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `Eq` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#Eq` | +| `NotEq` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#NotEq` | +| `Lt` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#Lt` | +| `LtE` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#LtE` | +| `Gt` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#Gt` | +| `GtE` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#GtE` | +| `Is` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#Is` | +| `IsNot` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#IsNot` | +| `In` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#In` | +| `NotIn` | | no fields | | | `Parser/Python.asdl:109@v3.15.0rc1#NotIn` | + +### 2.9 `comprehension` + +A single shape with 4 fields, declared at `Parser/Python.asdl:111@v3.15.0rc1#comprehension`. There is nothing to switch on: every value of this type has exactly these fields. + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `target` | `expr` | required | +| 2 | `iter` | `expr` | required | +| 3 | `ifs` | `expr` | sequence | +| 4 | `is_async` | `int` | required | + +### 2.10 `excepthandler` + +A choice between 1 constructors, declared at `Parser/Python.asdl:113@v3.15.0rc1#excepthandler`. + +Every `excepthandler` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | optional | +| `end_col_offset` | `int` | optional | + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `ExceptHandler` | 1 | `type` | `expr` | optional | `Parser/Python.asdl:113@v3.15.0rc1#ExceptHandler` | +| `ExceptHandler` | 2 | `name` | `identifier` | optional | | +| `ExceptHandler` | 3 | `body` | `stmt` | sequence | | + +### 2.11 `arguments` + +A single shape with 7 fields, declared at `Parser/Python.asdl:116@v3.15.0rc1#arguments`. There is nothing to switch on: every value of this type has exactly these fields. + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `posonlyargs` | `arg` | sequence | +| 2 | `args` | `arg` | sequence | +| 3 | `vararg` | `arg` | optional | +| 4 | `kwonlyargs` | `arg` | sequence | +| 5 | `kw_defaults` | `expr` | sequence of optional | +| 6 | `kwarg` | `arg` | optional | +| 7 | `defaults` | `expr` | sequence | + +### 2.12 `arg` + +A single shape with 3 fields, declared at `Parser/Python.asdl:119@v3.15.0rc1#arg`. There is nothing to switch on: every value of this type has exactly these fields. + +Every `arg` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | optional | +| `end_col_offset` | `int` | optional | + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `arg` | `identifier` | required | +| 2 | `annotation` | `expr` | optional | +| 3 | `type_comment` | `string` | optional | + +### 2.13 `keyword` + +A single shape with 2 fields, declared at `Parser/Python.asdl:123@v3.15.0rc1#keyword`. There is nothing to switch on: every value of this type has exactly these fields. + +Every `keyword` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | optional | +| `end_col_offset` | `int` | optional | + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `arg` | `identifier` | optional | +| 2 | `value` | `expr` | required | + +### 2.14 `alias` + +A single shape with 2 fields, declared at `Parser/Python.asdl:127@v3.15.0rc1#alias`. There is nothing to switch on: every value of this type has exactly these fields. + +Every `alias` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | optional | +| `end_col_offset` | `int` | optional | + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `name` | `identifier` | required | +| 2 | `asname` | `identifier` | optional | + +### 2.15 `withitem` + +A single shape with 2 fields, declared at `Parser/Python.asdl:130@v3.15.0rc1#withitem`. There is nothing to switch on: every value of this type has exactly these fields. + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `context_expr` | `expr` | required | +| 2 | `optional_vars` | `expr` | optional | + +### 2.16 `match_case` + +A single shape with 3 fields, declared at `Parser/Python.asdl:132@v3.15.0rc1#match_case`. There is nothing to switch on: every value of this type has exactly these fields. + +| Order | Field | Type | Holds | +|---|---|---|---| +| 1 | `pattern` | `pattern` | required | +| 2 | `guard` | `expr` | optional | +| 3 | `body` | `stmt` | sequence | + +### 2.17 `pattern` + +A choice between 8 constructors, declared at `Parser/Python.asdl:134@v3.15.0rc1#pattern`. + +Every `pattern` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | required | +| `end_col_offset` | `int` | required | + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `MatchValue` | 1 | `value` | `expr` | required | `Parser/Python.asdl:134@v3.15.0rc1#MatchValue` | +| `MatchSingleton` | 1 | `value` | `constant` | required | `Parser/Python.asdl:135@v3.15.0rc1#MatchSingleton` | +| `MatchSequence` | 1 | `patterns` | `pattern` | sequence | `Parser/Python.asdl:136@v3.15.0rc1#MatchSequence` | +| `MatchMapping` | 1 | `keys` | `expr` | sequence | `Parser/Python.asdl:137@v3.15.0rc1#MatchMapping` | +| `MatchMapping` | 2 | `patterns` | `pattern` | sequence | | +| `MatchMapping` | 3 | `rest` | `identifier` | optional | | +| `MatchClass` | 1 | `cls` | `expr` | required | `Parser/Python.asdl:138@v3.15.0rc1#MatchClass` | +| `MatchClass` | 2 | `patterns` | `pattern` | sequence | | +| `MatchClass` | 3 | `kwd_attrs` | `identifier` | sequence | | +| `MatchClass` | 4 | `kwd_patterns` | `pattern` | sequence | | +| `MatchStar` | 1 | `name` | `identifier` | optional | `Parser/Python.asdl:140@v3.15.0rc1#MatchStar` | +| `MatchAs` | 1 | `pattern` | `pattern` | optional | `Parser/Python.asdl:143@v3.15.0rc1#MatchAs` | +| `MatchAs` | 2 | `name` | `identifier` | optional | | +| `MatchOr` | 1 | `patterns` | `pattern` | sequence | `Parser/Python.asdl:144@v3.15.0rc1#MatchOr` | + +### 2.18 `type_ignore` + +A choice between 1 constructors, declared at `Parser/Python.asdl:148@v3.15.0rc1#type_ignore`. + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `TypeIgnore` | 1 | `lineno` | `int` | required | `Parser/Python.asdl:148@v3.15.0rc1#TypeIgnore` | +| `TypeIgnore` | 2 | `tag` | `string` | required | | + +### 2.19 `type_param` + +A choice between 3 constructors, declared at `Parser/Python.asdl:150@v3.15.0rc1#type_param`. + +Every `type_param` node also carries 4 attributes, which are not fields and are not part of the constructor's positional arguments. + +| Attribute | Type | Holds | +|---|---|---| +| `lineno` | `int` | required | +| `col_offset` | `int` | required | +| `end_lineno` | `int` | required | +| `end_col_offset` | `int` | required | + +| Node | Order | Field | Type | Holds | Declared at | +|---|---|---|---|---|---| +| `TypeVar` | 1 | `name` | `identifier` | required | `Parser/Python.asdl:150@v3.15.0rc1#TypeVar` | +| `TypeVar` | 2 | `bound` | `expr` | optional | | +| `TypeVar` | 3 | `default_value` | `expr` | optional | | +| `ParamSpec` | 1 | `name` | `identifier` | required | `Parser/Python.asdl:151@v3.15.0rc1#ParamSpec` | +| `ParamSpec` | 2 | `default_value` | `expr` | optional | | +| `TypeVarTuple` | 1 | `name` | `identifier` | required | `Parser/Python.asdl:152@v3.15.0rc1#TypeVarTuple` | +| `TypeVarTuple` | 2 | `default_value` | `expr` | optional | | + + +## 3. Algorithms + +There are only three algorithms here, and two of them are generated one per node kind. That is the honest shape of this subsystem: it is a data definition with a small amount of machinery around it. + +### 3.1 `make_node` + +**CPython:** `Python/Python-ast.c:7057-7089@v3.15.0rc1#_PyAST_FunctionDef` +**Precondition:** `arena` is the arena that will own the whole tree, every required field is non NULL +**Postcondition:** a node in the arena, with `kind` set and every field stored +**Complexity:** O(1) +**Fails:** returns NULL, sets an error + +One of these exists per constructor and they are all the same shape. `FunctionDef` is the example because it has all three kinds of field. + +``` +func make_FunctionDef(name: *PyObject, args: *Arguments, + body: *[*Stmt], decorator_list: *[*Expr], + returns: *Expr, type_comment: *PyObject, + type_params: *[*TypeParam], + lineno: int, col_offset: int, + end_lineno: int, end_col_offset: int, + arena: *Arena) -> *Stmt: + # Required fields are checked one at a time and named in the message. Optional + # fields are not checked, because NULL is what absent means for them. + if name == NULL: + set_error(ValueError, "field 'name' is required for FunctionDef") + return NULL + if args == NULL: + set_error(ValueError, "field 'args' is required for FunctionDef") + return NULL + let p: *Stmt = cast(*Stmt, arena_malloc(arena, sizeof(Stmt))) + if p == NULL: + return NULL + p->kind = FunctionDef_kind + p->v.FunctionDef.name = name + p->v.FunctionDef.args = args + p->v.FunctionDef.body = body + p->v.FunctionDef.decorator_list = decorator_list + p->v.FunctionDef.returns = returns + p->v.FunctionDef.type_comment = type_comment + p->v.FunctionDef.type_params = type_params + p->lineno = lineno + p->col_offset = col_offset + p->end_lineno = end_lineno + p->end_col_offset = end_col_offset + return p +``` + +Nothing is copied and nothing is incref'd. The caller hands over pointers it got from the arena and the node keeps them. A sequence field is stored as given, including `NULL`, which reads as empty everywhere downstream. + +Note what is missing: there is no check that a sequence is non empty, no check that a field holds the right kind of node, and no check that the location makes sense. Those are section 3.3's job and they only run on a tree that came from Python rather than from the parser. + +### 3.2 `new_sequence` + +**CPython:** `Include/internal/pycore_asdl.h:52-74@v3.15.0rc1#GENERATE_ASDL_SEQ_CONSTRUCTOR` +**Precondition:** `size` is not negative, `arena` is the tree's arena +**Postcondition:** a sequence of exactly `size` slots, in the arena +**Complexity:** O(1), one arena allocation +**Fails:** returns NULL, sets `MemoryError` + +``` +func new_sequence(size: ssize, arena: *Arena) -> *Seq: + if size < 0 or (size > 0 and cast(uint, size - 1) > SIZE_MAX / sizeof(*void)): + set_error(MemoryError) + return NULL + let n: uint = 0 + if size > 0: + n = sizeof(*Elem) * (size - 1) # one slot is already in the header + if n > SIZE_MAX - sizeof(Seq): + set_error(MemoryError) + return NULL + n = n + sizeof(Seq) + let seq: *Seq = cast(*Seq, arena_malloc(arena, n)) + if seq == NULL: + set_error(MemoryError) + return NULL + memset(seq, 0, n) # every slot starts NULL + seq->size = size + seq->elements = cast(**void, seq->typed_elements) + return seq +``` + +The two overflow checks are the whole reason this is a function and not a macro. A port on a 64 bit target where sizes come from a parser will never reach either of them from real source, and should keep them anyway, because the size is attacker controlled the moment a program calls `compile` on something it downloaded. + +The `memset` is load bearing. A caller allocates a sequence and then fills it, and anything it has not filled yet has to be `NULL` rather than whatever was in the arena, because a partly filled sequence is what the parser is holding when an alternative fails. + +The last line is the one to read twice. The struct has two views of the same memory: `typed_elements`, declared with the element type so `asdl_seq_GET` returns something the compiler knows about, and `elements`, a `void **` for the generic code that walks a sequence without caring what is in it. Setting one to point at the other is what makes both work, and the two are aliases and not copies. In a language with generics, one typed slice replaces both and the line disappears. + +### 3.3 `validate_tree` + +**CPython:** `Python/ast.c:1049-1076@v3.15.0rc1#_PyAST_Validate` +**Precondition:** `mod` is a tree, from the parser or built by a Python program +**Postcondition:** returns 1 when the tree can be compiled, 0 with an error set when it cannot +**Complexity:** O(nodes) +**Fails:** returns 0, sets `ValueError` or `TypeError` + +``` +func validate_tree(mod: *Mod) -> int: + if mod->kind == Module_kind: + return validate_statements(mod->v.Module.body) + if mod->kind == Interactive_kind: + return validate_statements(mod->v.Interactive.body) + if mod->kind == Expression_kind: + return validate_expression(mod->v.Expression.body, Load) + if mod->kind == FunctionType_kind: + if validate_expressions(mod->v.FunctionType.argtypes, Load, false) == 0: + return 0 + return validate_expression(mod->v.FunctionType.returns, Load) + set_error(SystemError, "impossible module node") + return 0 +``` + +The rules it applies below the top are structural and are not in the grammar. The one that catches most hand built trees is that a body may not be empty, at `Python/ast.c:704-711@v3.15.0rc1#_validate_nonempty_seq` and applied by `Python/ast.c:722-726@v3.15.0rc1#validate_body`. `stmt* body` in the grammar allows zero statements and `FunctionDef` with zero statements is rejected here, which is the clearest case of the grammar being necessary and not sufficient. + +Recursion is bounded by the interpreter's recursion limit rather than by a depth counter of its own, through the macro at `Python/ast.c:13-18@v3.15.0rc1#ENTER_RECURSIVE`. A tree deep enough to overflow is refused with `RecursionError` rather than crashing. + +## 4. Invariants + +**INV-AST-001.** The order of a constructor's fields in section 2 is the order of `_fields` on the corresponding Python class, and the order that class takes positional arguments in. Reordering two fields of the same type is undetectable at the C level and changes the meaning of every positional construction in every program. + +**INV-AST-002.** A required field is never `NULL` in a tree that has been validated. The constructor refuses to build the node, so a tree from the parser cannot violate this, and a tree from Python is checked before it is compiled. + +**INV-AST-003.** An optional field may be `NULL` and that is the only representation of absence. There is no second flag and no sentinel node. + +**INV-AST-004.** A sequence field is either a sequence or `NULL`, and `NULL` reads as length zero everywhere. An empty sequence and an absent one are not distinguishable and no code may try. + +**INV-AST-005.** Every node and every sequence in one tree belongs to one arena, and no pointer into that tree outlives it. The `PyObject *` fields are the exception: the arena holds a reference to each of them and releases it when the arena goes. + +**INV-AST-006.** Attributes are not fields. They are declared once per type, they appear on every constructor of that type, they are listed in `_attributes` rather than `_fields`, and they are not positional arguments. + +**INV-AST-007.** A node kind name is unique across the whole grammar, so a name identifies a constructor without saying which type it belongs to. This is what lets the C code use one flat `Xxx_kind` enum per sum and one flat namespace of Python classes. + +**INV-AST-008.** Every node kind of a type that carries the four location attributes has a start line and start column. The two end attributes are optional and may be absent on a tree built by hand. + +**INV-AST-009.** Validation is not idempotent with construction. A tree that a constructor accepted may still fail validation, because the constructor checks required fields and validation checks structure. + +## 5. Observable behaviour + + +The whole grammar is visible from Python. Each of the 113 node kinds below is a class in the `ast` module with the same name, each of the 19 types is a class those inherit from, and the field order in the grammar is the order those classes take positional arguments in. A reimplementation that renames a field or reorders two of them is detectable by any program that builds a tree by hand or reads one back. + +The three field kinds are three different things to leave out. A required field has to be passed, and building the node without it raises `TypeError` naming the field. An optional field left out is `None`. A sequence field left out is a new empty list, so `body` is `[]` rather than missing. There is one field type that breaks the pattern: a field of type `expr_context` left out is the `Load` singleton, because nearly every expression in a tree is being read rather than written to. + +The 4 location attributes are separate from the fields. They are listed in `_attributes` rather than `_fields`, and none of them is ever required by the constructor, so a node can always be built without them. The two declared optional default to `None` like any other optional. The two declared required have no value at all, and reading one raises `AttributeError` rather than returning `None`. Nothing complains until `compile` sees the tree, which is where a missing line number becomes `TypeError` and where a port has to put the same check. + +| Class | Base | `_fields` | `_attributes` | +|---|---|---|---| +| `mod` | abstract | `()` | `()` | +| `Module` | `mod` | `('body', 'type_ignores')` | `()` | +| `Interactive` | `mod` | `('body',)` | `()` | +| `Expression` | `mod` | `('body',)` | `()` | +| `FunctionType` | `mod` | `('argtypes', 'returns')` | `()` | +| `stmt` | abstract | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `FunctionDef` | `stmt` | `('name', 'args', 'body', 'decorator_list', 'returns', 'type_comment', 'type_params')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `AsyncFunctionDef` | `stmt` | `('name', 'args', 'body', 'decorator_list', 'returns', 'type_comment', 'type_params')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `ClassDef` | `stmt` | `('name', 'bases', 'keywords', 'body', 'decorator_list', 'type_params')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Return` | `stmt` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Delete` | `stmt` | `('targets',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Assign` | `stmt` | `('targets', 'value', 'type_comment')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `TypeAlias` | `stmt` | `('name', 'type_params', 'value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `AugAssign` | `stmt` | `('target', 'op', 'value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `AnnAssign` | `stmt` | `('target', 'annotation', 'value', 'simple')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `For` | `stmt` | `('target', 'iter', 'body', 'orelse', 'type_comment')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `AsyncFor` | `stmt` | `('target', 'iter', 'body', 'orelse', 'type_comment')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `While` | `stmt` | `('test', 'body', 'orelse')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `If` | `stmt` | `('test', 'body', 'orelse')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `With` | `stmt` | `('items', 'body', 'type_comment')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `AsyncWith` | `stmt` | `('items', 'body', 'type_comment')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Match` | `stmt` | `('subject', 'cases')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Raise` | `stmt` | `('exc', 'cause')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Try` | `stmt` | `('body', 'handlers', 'orelse', 'finalbody')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `TryStar` | `stmt` | `('body', 'handlers', 'orelse', 'finalbody')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Assert` | `stmt` | `('test', 'msg')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Import` | `stmt` | `('names', 'is_lazy')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `ImportFrom` | `stmt` | `('module', 'names', 'level', 'is_lazy')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Global` | `stmt` | `('names',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Nonlocal` | `stmt` | `('names',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Expr` | `stmt` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Pass` | `stmt` | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Break` | `stmt` | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Continue` | `stmt` | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `expr` | abstract | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `BoolOp` | `expr` | `('op', 'values')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `NamedExpr` | `expr` | `('target', 'value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `BinOp` | `expr` | `('left', 'op', 'right')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `UnaryOp` | `expr` | `('op', 'operand')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Lambda` | `expr` | `('args', 'body')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `IfExp` | `expr` | `('test', 'body', 'orelse')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Dict` | `expr` | `('keys', 'values')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Set` | `expr` | `('elts',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `ListComp` | `expr` | `('elt', 'generators')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `SetComp` | `expr` | `('elt', 'generators')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `DictComp` | `expr` | `('key', 'value', 'generators')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `GeneratorExp` | `expr` | `('elt', 'generators')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Await` | `expr` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Yield` | `expr` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `YieldFrom` | `expr` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Compare` | `expr` | `('left', 'ops', 'comparators')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Call` | `expr` | `('func', 'args', 'keywords')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `FormattedValue` | `expr` | `('value', 'conversion', 'format_spec')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Interpolation` | `expr` | `('value', 'str', 'conversion', 'format_spec')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `JoinedStr` | `expr` | `('values',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `TemplateStr` | `expr` | `('values',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Constant` | `expr` | `('value', 'kind')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Attribute` | `expr` | `('value', 'attr', 'ctx')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Subscript` | `expr` | `('value', 'slice', 'ctx')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Starred` | `expr` | `('value', 'ctx')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Name` | `expr` | `('id', 'ctx')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `List` | `expr` | `('elts', 'ctx')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Tuple` | `expr` | `('elts', 'ctx')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `Slice` | `expr` | `('lower', 'upper', 'step')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `expr_context` | abstract | `()` | `()` | +| `Load` | `expr_context` | `()` | `()` | +| `Store` | `expr_context` | `()` | `()` | +| `Del` | `expr_context` | `()` | `()` | +| `boolop` | abstract | `()` | `()` | +| `And` | `boolop` | `()` | `()` | +| `Or` | `boolop` | `()` | `()` | +| `operator` | abstract | `()` | `()` | +| `Add` | `operator` | `()` | `()` | +| `Sub` | `operator` | `()` | `()` | +| `Mult` | `operator` | `()` | `()` | +| `MatMult` | `operator` | `()` | `()` | +| `Div` | `operator` | `()` | `()` | +| `Mod` | `operator` | `()` | `()` | +| `Pow` | `operator` | `()` | `()` | +| `LShift` | `operator` | `()` | `()` | +| `RShift` | `operator` | `()` | `()` | +| `BitOr` | `operator` | `()` | `()` | +| `BitXor` | `operator` | `()` | `()` | +| `BitAnd` | `operator` | `()` | `()` | +| `FloorDiv` | `operator` | `()` | `()` | +| `unaryop` | abstract | `()` | `()` | +| `Invert` | `unaryop` | `()` | `()` | +| `Not` | `unaryop` | `()` | `()` | +| `UAdd` | `unaryop` | `()` | `()` | +| `USub` | `unaryop` | `()` | `()` | +| `cmpop` | abstract | `()` | `()` | +| `Eq` | `cmpop` | `()` | `()` | +| `NotEq` | `cmpop` | `()` | `()` | +| `Lt` | `cmpop` | `()` | `()` | +| `LtE` | `cmpop` | `()` | `()` | +| `Gt` | `cmpop` | `()` | `()` | +| `GtE` | `cmpop` | `()` | `()` | +| `Is` | `cmpop` | `()` | `()` | +| `IsNot` | `cmpop` | `()` | `()` | +| `In` | `cmpop` | `()` | `()` | +| `NotIn` | `cmpop` | `()` | `()` | +| `comprehension` | concrete | `('target', 'iter', 'ifs', 'is_async')` | `()` | +| `excepthandler` | abstract | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `ExceptHandler` | `excepthandler` | `('type', 'name', 'body')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `arguments` | concrete | `('posonlyargs', 'args', 'vararg', 'kwonlyargs', 'kw_defaults', 'kwarg', 'defaults')` | `()` | +| `arg` | concrete | `('arg', 'annotation', 'type_comment')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `keyword` | concrete | `('arg', 'value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `alias` | concrete | `('name', 'asname')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `withitem` | concrete | `('context_expr', 'optional_vars')` | `()` | +| `match_case` | concrete | `('pattern', 'guard', 'body')` | `()` | +| `pattern` | abstract | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchValue` | `pattern` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchSingleton` | `pattern` | `('value',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchSequence` | `pattern` | `('patterns',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchMapping` | `pattern` | `('keys', 'patterns', 'rest')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchClass` | `pattern` | `('cls', 'patterns', 'kwd_attrs', 'kwd_patterns')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchStar` | `pattern` | `('name',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchAs` | `pattern` | `('pattern', 'name')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `MatchOr` | `pattern` | `('patterns',)` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `type_ignore` | abstract | `()` | `()` | +| `TypeIgnore` | `type_ignore` | `('lineno', 'tag')` | `()` | +| `type_param` | abstract | `()` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `TypeVar` | `type_param` | `('name', 'bound', 'default_value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `ParamSpec` | `type_param` | `('name', 'default_value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | +| `TypeVarTuple` | `type_param` | `('name', 'default_value')` | `('lineno', 'col_offset', 'end_lineno', 'end_col_offset')` | + + +## 6. Edge cases and error paths + +### 6.1 Building a node with fields missing + +Omitting a field is allowed for some fields and not others, and which is which comes from the field's kind in section 2. The defaulting happens in `ast_type_init` at `Python/Python-ast.c:5266@v3.15.0rc1#ast_type_init`, in the block at `Python/Python-ast.c:5417-5446@v3.15.0rc1`. + +An omitted optional field is `None`. An omitted sequence field is a new empty list. An omitted `expr_context` field is the `Load` singleton, which is a special case with no equivalent anywhere else in the grammar. An omitted field of any other kind is missing, and the constructor raises `TypeError` naming all of them at once, at `Python/Python-ast.c:5448-5459@v3.15.0rc1`. + +So `ast.FunctionDef()` raises `TypeError` for `name` and `args` together, and `ast.FunctionDef(name="f", args=ast.arguments())` succeeds with `body` an empty list and `returns` set to `None`. The second one is a node that exists and cannot be compiled, which is section 6.2. + +### 6.2 A tree that is valid by the grammar and rejected anyway + +`stmt* body` permits zero statements. Every construct with a body rejects zero statements at validation time with `ValueError: empty body on FunctionDef`. The same applies to `Delete` with no targets, `Assign` with no targets, `With` with no items and `Match` with no cases. + +A port that generates its node types from the grammar and stops there will accept trees CPython refuses. The grammar is the vocabulary and validation is the grammar of the vocabulary, and only one of the two is generated. + +### 6.3 Depth + +There is no limit on tree depth in the grammar or in the node structures. The limits are the parser's, and then the recursion limit during validation, symbol table construction and code generation. A tree deep enough raises `RecursionError` at whichever of those reaches it first, so the same program can fail at different stages depending on the recursion limit in force. + +### 6.4 Allocation failure + +Every constructor and every sequence allocation can fail and returns `NULL` with `MemoryError` set. There is no cleanup on that path and there does not need to be, because everything allocated so far is in the arena and the arena is freed by whoever created it. This is the single largest simplification the arena buys and it is worth keeping in a port. + +### 6.5 A tree that came from Python + +`PyAST_obj2mod` at `Python/Python-ast.c:18493-18499@v3.15.0rc1#PyAST_obj2mod` converts Python objects back into C nodes, and it is the only way a tree that the parser did not build gets into the compiler. It raises an audit event before it does anything else, because at that point a program is handing the compiler a structure that no source text produced. + +Every type error a hand built tree can contain is caught here or in validation. A field holding the wrong kind of node, a string where an `identifier` belongs, a list where a single node belongs: all of them are conversion failures with a message naming the field. + +The attributes are checked here too, and this is where INV-AST-008 is enforced rather than at construction time. `ast.Pass()` builds fine and has no `lineno`, and `compile` on a module containing it raises `TypeError: required field "lineno" missing from stmt`. `end_lineno` and `end_col_offset` are `int?` in the grammar and a node without them compiles, which is the difference between the two pairs and the reason only two of the four are declared optional. + +### 6.6 The four builtin types + +`identifier` and `string` are both `PyObject *` holding a `str`, and nothing at the C level tells them apart. `identifier` is interned and `string` is not, which matters for the speed of the symbol table lookups downstream rather than for correctness. `constant` is any Python object and is the one field type with no constraint at all, which is why `Constant.value` can hold anything a literal can produce. `int` is a C `int` and not a Python integer, so a field like `AnnAssign.simple` is a flag with a C sized range. + +### 6.7 The two sequences whose slots can be empty + +Two fields in the grammar are written `expr?*`, with both quantifiers. They are sequences, and the entries in them are allowed to be `None`. Everything else in the grammar is a sequence or optional, never both, so these two are worth naming. + +`Dict` declares `expr?* keys` at `Parser/Python.asdl:66@v3.15.0rc1#Dict`. `keys` and `values` run in parallel, one slot per entry in the display, and a `None` key means the entry was a `**` unpacking with the mapping in `values`. So `{1: 2, **d}` parses to `keys=[Constant(1), None]` and `values=[Constant(2), Name('d')]`. There is no separate node for dictionary unpacking. The `None` is the node. + +`arguments` declares `expr?* kw_defaults` at `Parser/Python.asdl:117@v3.15.0rc1#kw_defaults`. It runs in parallel with `kwonlyargs` rather than listing only the defaults that exist, so the default for `kwonlyargs[i]` is always `kw_defaults[i]`, and it is `None` when that argument has no default. `def f(*, a, b=1)` gives `kw_defaults=[None, Constant(1)]`. `defaults`, which covers the positional arguments, works the other way: it is right aligned against `args` and holds no gaps, because a positional argument with a default cannot be followed by one without. + +This is easy to lose in a port, because `asdl.py` itself nearly loses it. `Field.seq` and `Field.opt` are set from the last quantifier only, so `expr?*` arrives with `seq` true and `opt` false, exactly like `expr*`. The `?` survives in `Field.quantifiers` and nowhere else. A port that reads `seq` and `opt` types both fields as plain lists of expressions, and then crashes on the first `{**d}` and the first keyword only argument without a default. + +## 7. Interactions + +`BP-PARSER` produces the tree and is the only producer other than a Python program calling `compile` on an `ast` object. It builds nodes bottom up with the constructors in section 3.1, into the arena it was given. + +`BP-SYMTABLE` walks the tree once and reads names, and it depends on field order only through the field names, not their positions. It is the first consumer to reject trees that validation accepted, because scoping rules are not structural. + +`BP-CODEGEN` walks the tree and reads the location attributes on every node to build the line table. INV-AST-008 is the one this depends on: a node with no start line produces a code object whose line table is wrong, and a wrong line table is only detectable in a traceback, which is the worst place to find out. + +The `ast` module is the public face of everything here and is the reason this subsystem is tier B rather than tier D. `ast.parse`, `ast.dump`, `ast.NodeVisitor` and `ast.unparse` are all built on `_fields` and `_attributes` being exactly what section 5 says they are. + +`BP-PIPELINE` fixes when the arena is created and freed, which is what makes INV-AST-005 hold in practice rather than by convention. + +## 8. Conformance + + +Sections 1, 2 and 5 are generated from the grammar file, so the way they go wrong is not a typo. They go wrong when the running interpreter and the pinned grammar have stopped agreeing, which is what the checks below are for. Each one reads the grammar from the pinned tree and compares it against the `ast` module of the interpreter running the test. + +| Claim | Held up by | Covers | +|---|---|---| +| Every type in section 1 is a class in `ast` | `test_every_type_in_the_grammar_is_a_class_in_ast` | 19 types | +| Every constructor in section 2 is a class in `ast` | `test_every_constructor_in_the_grammar_is_a_class_in_ast` | 113 node kinds | +| `_fields` is the grammar's field names, in the grammar's order | `test_the_field_order_is_the_order_the_grammar_declares` | 198 fields | +| `_attributes` is the grammar's attributes, in the grammar's order | `test_the_attributes_are_the_ones_the_grammar_declares` | 8 types carry attributes | +| Leaving a field out does what section 5 says it does | `test_the_defaults_are_the_ones_section_5_describes` | the three field kinds | +| Every citation generated into sections 1 and 2 resolves against the pinned tree | `just citations` | 145 citations | + +The first five run under `just test` and live in `tools/bpc/tests/test_bpc_conformance.py`. They are skipped on an interpreter whose version does not match the pinned tree, because a difference between v3.15.0rc1 and some other version is a fact about the two versions rather than a failure of this document. + + +## 9. Port notes + +Generate the node types rather than typing them. The grammar file is 154 lines and it is stable across releases in the parts that matter, so a port that reads it produces the same 113 node kinds with the same field names in the same order, and stays right when the pin moves. A port that types them by hand is doing the one job in this subsystem that a machine does better. + +In Go the natural shape is one interface per sum type and one struct per constructor, with a marker method. That gives type switches where CPython has a `kind` enum, and it loses nothing, because CPython's union is a closed set and so is the interface. The cost is that a nil interface value and an interface holding a nil pointer are different things, and optional fields are exactly where that bites. Use a pointer field and check for nil, never an interface. + +In Rust the natural shape is one enum per sum type with a variant per constructor, which is closer to the C than Go gets and gives exhaustive matching for free. `Option` for optional fields and `Vec` for sequences map exactly, with one caveat: `Vec` cannot be absent, and CPython's `NULL` sequence has to become an empty `Vec` at the boundary rather than an `Option>`. Making that distinction representable is how INV-AST-004 gets violated by accident. + +The arena is worth keeping in both. In Rust an arena crate or a `Vec` of nodes with index handles removes the lifetime problem that a tree of `Box`es creates during parsing, when a failed alternative discards a subtree. In Go the garbage collector already does this, so an arena buys allocation speed rather than correctness, and can be skipped until it is measured. + +Validation cannot be generated and has to be written. It is a few hundred lines and it is the difference between a port that compiles the same programs CPython compiles and one that accepts trees CPython refuses. Start with the non empty body rule, which is the one every real program depends on without knowing it. + +The location attributes are not optional in practice. A port that leaves them out has a working compiler and unusable tracebacks, and adding them afterwards means touching every constructor call in the parser. diff --git a/blueprints/README.md b/blueprints/README.md index c371830..de6ea89 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -57,14 +57,24 @@ The tier says how closely a reimplementation has to match this subsystem for a p | Blueprint | Covers | Lesson | Status | |---|---|---|---| +| [BP-AST](BP-AST.md) | the node types, and every field of every one of them | T03 | partial | | [BP-MAP](BP-MAP.md) | the architecture every other blueprint hangs off | T10 | complete | | [BP-PIPELINE](BP-PIPELINE.md) | source text to a running frame, as a contract | T01 | complete | -Thirty more are planned, one per subsystem, listed in the milestone issues. The two here are the ones the rest depend on: `BP-MAP` names the boundaries so that two blueprints cannot both claim the same code, and `BP-PIPELINE` fixes the stage list and what crosses between the stages. +Thirty more are planned, one per subsystem, listed in the milestone issues. `BP-MAP` and `BP-PIPELINE` are the two the rest depend on: `BP-MAP` names the boundaries so that two blueprints cannot both claim the same code, and `BP-PIPELINE` fixes the stage list and what crosses between the stages. + +## The generated ones + +`BP-AST` is half written and half compiled. Its sections 1, 2 and 5 are a table of 19 types, 113 node kinds and 198 fields, which is transcription, and transcription is right the day it is typed and wrong the first time upstream adds a field. So those sections are generated from `Parser/Python.asdl` by [bpc](../tools/bpc), and the file that gets edited is [sources/BP-AST.md](sources/BP-AST.md), which holds the prose with a one line directive where each generated block goes. + +Both files are committed. `blueprints/BP-AST.md` is what people read and what the checks lint, with `` and `` around each generated part so the boundary is visible. Editing between those markers is pointless: `just blueprints` fails on the next run and `just build-blueprints` throws the edit away. + +A blueprint with no source document under `sources/` is entirely hand written, which is most of them and will stay that way. Generating a section is worth it only where upstream ships the material in a form a program can read, and `Parser/Python.asdl` is the clearest case of that in CPython. ## Checking them ``` -just blueprints # the nine sections, the header block, the invariant IDs -just citations # every citation resolves against the pinned tree +just blueprints # the nine sections, the header block, the invariant IDs +just citations # every citation resolves against the pinned tree +just build-blueprints # regenerate the compiled sections after the pin moves ``` diff --git a/blueprints/sources/BP-AST.md b/blueprints/sources/BP-AST.md new file mode 100644 index 0000000..c624a33 --- /dev/null +++ b/blueprints/sources/BP-AST.md @@ -0,0 +1,264 @@ +# BP-AST: the abstract syntax tree + +**Covers:** `Parser/Python.asdl`, `Python/Python-ast.c`, `Python/ast.c`, `Include/internal/pycore_ast.h` and `Include/internal/pycore_asdl.h`, at the pinned tag +**Lesson:** T03, tokens become a tree +**Status:** partial +**Compatibility tier:** B + +## 1. Purpose and scope + +This blueprint specifies the shape of the tree the parser produces and the compiler consumes. It names every node kind, every field of every node kind, the order those fields are in, whether each one is required, optional or a sequence, and which node kinds carry source locations. + +In scope: the node vocabulary, the C representation of it, how a node is allocated and how long it lives, what a Python program can see of all this through the `ast` module, and the structural checks CPython applies to a tree that was built by hand rather than parsed. + +Out of scope: how the tree gets built. The grammar and the parser that matches it are `BP-PARSER`, and this blueprint says nothing about which source text produces which node. What happens to the tree afterwards is `BP-SYMTABLE` and `BP-CODEGEN`. The order the stages run in is `BP-PIPELINE`. + +The table at the end of this section, the whole of section 2, the whole of section 5 and the whole of section 8 are generated from `Parser/Python.asdl` by `bpc`, the same file CPython generates its own node structures, its C constructors and its Python classes from. They are not typed by anybody and they are not proofread by anybody. Everything else in this document is written by hand, including the paragraphs above, because where this subsystem stops and `BP-PARSER` starts is not in the grammar and never will be. A generated table cannot be one field out of date, which is the way a hand written one goes wrong: correct on the day it is written, wrong the first time upstream adds a field, and nobody finds out because a reader who trusted the table has no reason to check it. + +The generator is `Parser/asdl_c.py:1-2@v3.15.0rc1`, run from the `regen-ast` rule at `Makefile.pre.in:2056-2072@v3.15.0rc1`, which writes three files: the node structures, the per interpreter state that holds the Python classes, and the C source that builds both. Everything this blueprint describes is downstream of one 154 line grammar file. + + + +## 2. Data structures + +Every node kind in the grammar becomes one C struct, one Python class and one constructor function. This section lists what the grammar says. How that becomes C is here, and the tables below are the specification of what a port has to build. + +### 2.0.1 The C shape of a node + +A sum type becomes a struct holding a `kind` enum and a union of one anonymous struct per constructor, so `struct _stmt` at `Include/internal/pycore_ast.h:196-207@v3.15.0rc1#_stmt` has a `kind` of `FunctionDef_kind` and a `v.FunctionDef` holding that constructor's seven fields. A product type has no `kind` and no union, because there is nothing to switch on. + +The attributes listed for a type sit outside the union, once, rather than being repeated in each arm. That is what makes `node->lineno` readable without knowing which kind of statement it is, and it is the reason attributes and fields are separate concepts rather than a naming convention. + +A field whose type is one of the four ASDL built ins becomes a `PyObject *` for `identifier`, `string` and `constant`, and a plain `int` for `int`. A field whose type is another node type becomes a pointer to that node's struct. An optional field is the same pointer and may be `NULL`. There is no separate tag saying whether an optional field is present. + +### 2.0.2 Sequences + +A sequence field is a pointer to an `asdl_seq`, which is a length and an array of pointers laid out inline after it. The header is two words, at `Include/internal/pycore_asdl.h:24-26@v3.15.0rc1#_ASDL_SEQ_HEAD`, and one typed variant is generated per element type by the macro at `Include/internal/pycore_asdl.h:52-74@v3.15.0rc1#GENERATE_ASDL_SEQ_CONSTRUCTOR`. + +The length and the elements are read through macros rather than directly, and `asdl_seq_LEN` at `Include/internal/pycore_asdl.h:83@v3.15.0rc1#asdl_seq_LEN` reports zero for a `NULL` sequence. A port that represents a sequence as a growable array gets this for free. A port that distinguishes an absent list from an empty one has invented a state CPython does not have. + +Sequences are allocated at their final length and never grown. The parser knows how many children it matched before it builds the node, so there is no append path and no capacity field. + +### 2.0.3 Lifetime + +Every node and every sequence is allocated from the compile time arena, through `_PyArena_Malloc` at `Include/internal/pycore_pyarena.h:56@v3.15.0rc1#_PyArena_Malloc`, and nothing is freed individually. The whole tree goes away in one call when the arena is released, which happens once a code object exists. + +This is why no node has a destructor and why nothing in the tree is reference counted except the `PyObject *` fields, which the arena holds a reference to on the tree's behalf. A port that allocates nodes individually has to answer a question CPython never asks: who owns a subtree that was built and then discarded when a parser alternative failed. The arena's answer is that nobody does and it does not matter. + + + +## 3. Algorithms + +There are only three algorithms here, and two of them are generated one per node kind. That is the honest shape of this subsystem: it is a data definition with a small amount of machinery around it. + +### 3.1 `make_node` + +**CPython:** `Python/Python-ast.c:7057-7089@v3.15.0rc1#_PyAST_FunctionDef` +**Precondition:** `arena` is the arena that will own the whole tree, every required field is non NULL +**Postcondition:** a node in the arena, with `kind` set and every field stored +**Complexity:** O(1) +**Fails:** returns NULL, sets an error + +One of these exists per constructor and they are all the same shape. `FunctionDef` is the example because it has all three kinds of field. + +``` +func make_FunctionDef(name: *PyObject, args: *Arguments, + body: *[*Stmt], decorator_list: *[*Expr], + returns: *Expr, type_comment: *PyObject, + type_params: *[*TypeParam], + lineno: int, col_offset: int, + end_lineno: int, end_col_offset: int, + arena: *Arena) -> *Stmt: + # Required fields are checked one at a time and named in the message. Optional + # fields are not checked, because NULL is what absent means for them. + if name == NULL: + set_error(ValueError, "field 'name' is required for FunctionDef") + return NULL + if args == NULL: + set_error(ValueError, "field 'args' is required for FunctionDef") + return NULL + let p: *Stmt = cast(*Stmt, arena_malloc(arena, sizeof(Stmt))) + if p == NULL: + return NULL + p->kind = FunctionDef_kind + p->v.FunctionDef.name = name + p->v.FunctionDef.args = args + p->v.FunctionDef.body = body + p->v.FunctionDef.decorator_list = decorator_list + p->v.FunctionDef.returns = returns + p->v.FunctionDef.type_comment = type_comment + p->v.FunctionDef.type_params = type_params + p->lineno = lineno + p->col_offset = col_offset + p->end_lineno = end_lineno + p->end_col_offset = end_col_offset + return p +``` + +Nothing is copied and nothing is incref'd. The caller hands over pointers it got from the arena and the node keeps them. A sequence field is stored as given, including `NULL`, which reads as empty everywhere downstream. + +Note what is missing: there is no check that a sequence is non empty, no check that a field holds the right kind of node, and no check that the location makes sense. Those are section 3.3's job and they only run on a tree that came from Python rather than from the parser. + +### 3.2 `new_sequence` + +**CPython:** `Include/internal/pycore_asdl.h:52-74@v3.15.0rc1#GENERATE_ASDL_SEQ_CONSTRUCTOR` +**Precondition:** `size` is not negative, `arena` is the tree's arena +**Postcondition:** a sequence of exactly `size` slots, in the arena +**Complexity:** O(1), one arena allocation +**Fails:** returns NULL, sets `MemoryError` + +``` +func new_sequence(size: ssize, arena: *Arena) -> *Seq: + if size < 0 or (size > 0 and cast(uint, size - 1) > SIZE_MAX / sizeof(*void)): + set_error(MemoryError) + return NULL + let n: uint = 0 + if size > 0: + n = sizeof(*Elem) * (size - 1) # one slot is already in the header + if n > SIZE_MAX - sizeof(Seq): + set_error(MemoryError) + return NULL + n = n + sizeof(Seq) + let seq: *Seq = cast(*Seq, arena_malloc(arena, n)) + if seq == NULL: + set_error(MemoryError) + return NULL + memset(seq, 0, n) # every slot starts NULL + seq->size = size + seq->elements = cast(**void, seq->typed_elements) + return seq +``` + +The two overflow checks are the whole reason this is a function and not a macro. A port on a 64 bit target where sizes come from a parser will never reach either of them from real source, and should keep them anyway, because the size is attacker controlled the moment a program calls `compile` on something it downloaded. + +The `memset` is load bearing. A caller allocates a sequence and then fills it, and anything it has not filled yet has to be `NULL` rather than whatever was in the arena, because a partly filled sequence is what the parser is holding when an alternative fails. + +The last line is the one to read twice. The struct has two views of the same memory: `typed_elements`, declared with the element type so `asdl_seq_GET` returns something the compiler knows about, and `elements`, a `void **` for the generic code that walks a sequence without caring what is in it. Setting one to point at the other is what makes both work, and the two are aliases and not copies. In a language with generics, one typed slice replaces both and the line disappears. + +### 3.3 `validate_tree` + +**CPython:** `Python/ast.c:1049-1076@v3.15.0rc1#_PyAST_Validate` +**Precondition:** `mod` is a tree, from the parser or built by a Python program +**Postcondition:** returns 1 when the tree can be compiled, 0 with an error set when it cannot +**Complexity:** O(nodes) +**Fails:** returns 0, sets `ValueError` or `TypeError` + +``` +func validate_tree(mod: *Mod) -> int: + if mod->kind == Module_kind: + return validate_statements(mod->v.Module.body) + if mod->kind == Interactive_kind: + return validate_statements(mod->v.Interactive.body) + if mod->kind == Expression_kind: + return validate_expression(mod->v.Expression.body, Load) + if mod->kind == FunctionType_kind: + if validate_expressions(mod->v.FunctionType.argtypes, Load, false) == 0: + return 0 + return validate_expression(mod->v.FunctionType.returns, Load) + set_error(SystemError, "impossible module node") + return 0 +``` + +The rules it applies below the top are structural and are not in the grammar. The one that catches most hand built trees is that a body may not be empty, at `Python/ast.c:704-711@v3.15.0rc1#_validate_nonempty_seq` and applied by `Python/ast.c:722-726@v3.15.0rc1#validate_body`. `stmt* body` in the grammar allows zero statements and `FunctionDef` with zero statements is rejected here, which is the clearest case of the grammar being necessary and not sufficient. + +Recursion is bounded by the interpreter's recursion limit rather than by a depth counter of its own, through the macro at `Python/ast.c:13-18@v3.15.0rc1#ENTER_RECURSIVE`. A tree deep enough to overflow is refused with `RecursionError` rather than crashing. + +## 4. Invariants + +**INV-AST-001.** The order of a constructor's fields in section 2 is the order of `_fields` on the corresponding Python class, and the order that class takes positional arguments in. Reordering two fields of the same type is undetectable at the C level and changes the meaning of every positional construction in every program. + +**INV-AST-002.** A required field is never `NULL` in a tree that has been validated. The constructor refuses to build the node, so a tree from the parser cannot violate this, and a tree from Python is checked before it is compiled. + +**INV-AST-003.** An optional field may be `NULL` and that is the only representation of absence. There is no second flag and no sentinel node. + +**INV-AST-004.** A sequence field is either a sequence or `NULL`, and `NULL` reads as length zero everywhere. An empty sequence and an absent one are not distinguishable and no code may try. + +**INV-AST-005.** Every node and every sequence in one tree belongs to one arena, and no pointer into that tree outlives it. The `PyObject *` fields are the exception: the arena holds a reference to each of them and releases it when the arena goes. + +**INV-AST-006.** Attributes are not fields. They are declared once per type, they appear on every constructor of that type, they are listed in `_attributes` rather than `_fields`, and they are not positional arguments. + +**INV-AST-007.** A node kind name is unique across the whole grammar, so a name identifies a constructor without saying which type it belongs to. This is what lets the C code use one flat `Xxx_kind` enum per sum and one flat namespace of Python classes. + +**INV-AST-008.** Every node kind of a type that carries the four location attributes has a start line and start column. The two end attributes are optional and may be absent on a tree built by hand. + +**INV-AST-009.** Validation is not idempotent with construction. A tree that a constructor accepted may still fail validation, because the constructor checks required fields and validation checks structure. + +## 5. Observable behaviour + + + +## 6. Edge cases and error paths + +### 6.1 Building a node with fields missing + +Omitting a field is allowed for some fields and not others, and which is which comes from the field's kind in section 2. The defaulting happens in `ast_type_init` at `Python/Python-ast.c:5266@v3.15.0rc1#ast_type_init`, in the block at `Python/Python-ast.c:5417-5446@v3.15.0rc1`. + +An omitted optional field is `None`. An omitted sequence field is a new empty list. An omitted `expr_context` field is the `Load` singleton, which is a special case with no equivalent anywhere else in the grammar. An omitted field of any other kind is missing, and the constructor raises `TypeError` naming all of them at once, at `Python/Python-ast.c:5448-5459@v3.15.0rc1`. + +So `ast.FunctionDef()` raises `TypeError` for `name` and `args` together, and `ast.FunctionDef(name="f", args=ast.arguments())` succeeds with `body` an empty list and `returns` set to `None`. The second one is a node that exists and cannot be compiled, which is section 6.2. + +### 6.2 A tree that is valid by the grammar and rejected anyway + +`stmt* body` permits zero statements. Every construct with a body rejects zero statements at validation time with `ValueError: empty body on FunctionDef`. The same applies to `Delete` with no targets, `Assign` with no targets, `With` with no items and `Match` with no cases. + +A port that generates its node types from the grammar and stops there will accept trees CPython refuses. The grammar is the vocabulary and validation is the grammar of the vocabulary, and only one of the two is generated. + +### 6.3 Depth + +There is no limit on tree depth in the grammar or in the node structures. The limits are the parser's, and then the recursion limit during validation, symbol table construction and code generation. A tree deep enough raises `RecursionError` at whichever of those reaches it first, so the same program can fail at different stages depending on the recursion limit in force. + +### 6.4 Allocation failure + +Every constructor and every sequence allocation can fail and returns `NULL` with `MemoryError` set. There is no cleanup on that path and there does not need to be, because everything allocated so far is in the arena and the arena is freed by whoever created it. This is the single largest simplification the arena buys and it is worth keeping in a port. + +### 6.5 A tree that came from Python + +`PyAST_obj2mod` at `Python/Python-ast.c:18493-18499@v3.15.0rc1#PyAST_obj2mod` converts Python objects back into C nodes, and it is the only way a tree that the parser did not build gets into the compiler. It raises an audit event before it does anything else, because at that point a program is handing the compiler a structure that no source text produced. + +Every type error a hand built tree can contain is caught here or in validation. A field holding the wrong kind of node, a string where an `identifier` belongs, a list where a single node belongs: all of them are conversion failures with a message naming the field. + +The attributes are checked here too, and this is where INV-AST-008 is enforced rather than at construction time. `ast.Pass()` builds fine and has no `lineno`, and `compile` on a module containing it raises `TypeError: required field "lineno" missing from stmt`. `end_lineno` and `end_col_offset` are `int?` in the grammar and a node without them compiles, which is the difference between the two pairs and the reason only two of the four are declared optional. + +### 6.6 The four builtin types + +`identifier` and `string` are both `PyObject *` holding a `str`, and nothing at the C level tells them apart. `identifier` is interned and `string` is not, which matters for the speed of the symbol table lookups downstream rather than for correctness. `constant` is any Python object and is the one field type with no constraint at all, which is why `Constant.value` can hold anything a literal can produce. `int` is a C `int` and not a Python integer, so a field like `AnnAssign.simple` is a flag with a C sized range. + +### 6.7 The two sequences whose slots can be empty + +Two fields in the grammar are written `expr?*`, with both quantifiers. They are sequences, and the entries in them are allowed to be `None`. Everything else in the grammar is a sequence or optional, never both, so these two are worth naming. + +`Dict` declares `expr?* keys` at `Parser/Python.asdl:66@v3.15.0rc1#Dict`. `keys` and `values` run in parallel, one slot per entry in the display, and a `None` key means the entry was a `**` unpacking with the mapping in `values`. So `{1: 2, **d}` parses to `keys=[Constant(1), None]` and `values=[Constant(2), Name('d')]`. There is no separate node for dictionary unpacking. The `None` is the node. + +`arguments` declares `expr?* kw_defaults` at `Parser/Python.asdl:117@v3.15.0rc1#kw_defaults`. It runs in parallel with `kwonlyargs` rather than listing only the defaults that exist, so the default for `kwonlyargs[i]` is always `kw_defaults[i]`, and it is `None` when that argument has no default. `def f(*, a, b=1)` gives `kw_defaults=[None, Constant(1)]`. `defaults`, which covers the positional arguments, works the other way: it is right aligned against `args` and holds no gaps, because a positional argument with a default cannot be followed by one without. + +This is easy to lose in a port, because `asdl.py` itself nearly loses it. `Field.seq` and `Field.opt` are set from the last quantifier only, so `expr?*` arrives with `seq` true and `opt` false, exactly like `expr*`. The `?` survives in `Field.quantifiers` and nowhere else. A port that reads `seq` and `opt` types both fields as plain lists of expressions, and then crashes on the first `{**d}` and the first keyword only argument without a default. + +## 7. Interactions + +`BP-PARSER` produces the tree and is the only producer other than a Python program calling `compile` on an `ast` object. It builds nodes bottom up with the constructors in section 3.1, into the arena it was given. + +`BP-SYMTABLE` walks the tree once and reads names, and it depends on field order only through the field names, not their positions. It is the first consumer to reject trees that validation accepted, because scoping rules are not structural. + +`BP-CODEGEN` walks the tree and reads the location attributes on every node to build the line table. INV-AST-008 is the one this depends on: a node with no start line produces a code object whose line table is wrong, and a wrong line table is only detectable in a traceback, which is the worst place to find out. + +The `ast` module is the public face of everything here and is the reason this subsystem is tier B rather than tier D. `ast.parse`, `ast.dump`, `ast.NodeVisitor` and `ast.unparse` are all built on `_fields` and `_attributes` being exactly what section 5 says they are. + +`BP-PIPELINE` fixes when the arena is created and freed, which is what makes INV-AST-005 hold in practice rather than by convention. + +## 8. Conformance + + + +## 9. Port notes + +Generate the node types rather than typing them. The grammar file is 154 lines and it is stable across releases in the parts that matter, so a port that reads it produces the same 113 node kinds with the same field names in the same order, and stays right when the pin moves. A port that types them by hand is doing the one job in this subsystem that a machine does better. + +In Go the natural shape is one interface per sum type and one struct per constructor, with a marker method. That gives type switches where CPython has a `kind` enum, and it loses nothing, because CPython's union is a closed set and so is the interface. The cost is that a nil interface value and an interface holding a nil pointer are different things, and optional fields are exactly where that bites. Use a pointer field and check for nil, never an interface. + +In Rust the natural shape is one enum per sum type with a variant per constructor, which is closer to the C than Go gets and gives exhaustive matching for free. `Option` for optional fields and `Vec` for sequences map exactly, with one caveat: `Vec` cannot be absent, and CPython's `NULL` sequence has to become an empty `Vec` at the boundary rather than an `Option>`. Making that distinction representable is how INV-AST-004 gets violated by accident. + +The arena is worth keeping in both. In Rust an arena crate or a `Vec` of nodes with index handles removes the lifetime problem that a tree of `Box`es creates during parsing, when a failed alternative discards a subtree. In Go the garbage collector already does this, so an arena buys allocation speed rather than correctness, and can be skipped until it is measured. + +Validation cannot be generated and has to be written. It is a few hundred lines and it is the difference between a port that compiles the same programs CPython compiles and one that accepts trees CPython refuses. Start with the non empty body rule, which is the one every real program depends on without knowing it. + +The location attributes are not optional in practice. A port that leaves them out has a working compiler and unusable tracebacks, and adding them afterwards means touching every constructor call in the parser. diff --git a/citations.lock.json b/citations.lock.json index 01276a1..e75ba94 100644 --- a/citations.lock.json +++ b/citations.lock.json @@ -35,6 +35,26 @@ "first_line": "struct _ts {", "lines": 36 }, + "Include/internal/pycore_asdl.h:24-26@v3.15.0rc1": { + "digest": "4ddf5b195dc099ab", + "first_line": "#define _ASDL_SEQ_HEAD \\", + "lines": 3 + }, + "Include/internal/pycore_asdl.h:52-74@v3.15.0rc1": { + "digest": "c2a1fc8439ab8ce8", + "first_line": "#define GENERATE_ASDL_SEQ_CONSTRUCTOR(NAME, TYPE) \\", + "lines": 23 + }, + "Include/internal/pycore_asdl.h:83-83@v3.15.0rc1": { + "digest": "2e827a581aa9b590", + "first_line": "#define asdl_seq_LEN(S) _Py_RVALUE(((S) == NULL ? 0 : (S)->size))", + "lines": 1 + }, + "Include/internal/pycore_ast.h:196-207@v3.15.0rc1": { + "digest": "a282830350595a4f", + "first_line": "struct _stmt {", + "lines": 12 + }, "Include/internal/pycore_interp_structs.h:271-286@v3.15.0rc1": { "digest": "84ddd939ee6f307f", "first_line": "#ifndef Py_GIL_DISABLED", @@ -85,6 +105,11 @@ "first_line": "/* Values used as the oparg for LOAD_COMMON_CONSTANT */", "lines": 14 }, + "Include/internal/pycore_pyarena.h:56-56@v3.15.0rc1": { + "digest": "c6671958ae6c8ac8", + "first_line": "PyAPI_FUNC(void*) _PyArena_Malloc(PyArena *, size_t size);", + "lines": 1 + }, "Include/internal/pycore_runtime_structs.h:134-170@v3.15.0rc1": { "digest": "ada3da14c486fca9", "first_line": "struct pyruntimestate {", @@ -220,6 +245,11 @@ "first_line": "def unparse(ast_obj):", "lines": 7 }, + "Makefile.pre.in:2056-2072@v3.15.0rc1": { + "digest": "38e8e503bef0fb6b", + "first_line": ".PHONY: regen-ast", + "lines": 17 + }, "Modules/main.c:830-852@v3.15.0rc1": { "digest": "b15e6301c9c95772", "first_line": "int", @@ -295,21 +325,456 @@ "first_line": "/* This function is called by the tp_dealloc handler to clear weak references.", "lines": 24 }, + "Parser/Python.asdl:100-100@v3.15.0rc1": { + "digest": "ea36297e42edfc56", + "first_line": "expr_context = Load | Store | Del", + "lines": 1 + }, + "Parser/Python.asdl:102-102@v3.15.0rc1": { + "digest": "2df541e4a0acabb8", + "first_line": "boolop = And | Or", + "lines": 1 + }, + "Parser/Python.asdl:104-104@v3.15.0rc1": { + "digest": "1947b9e3e6f9b81c", + "first_line": "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift", + "lines": 1 + }, "Parser/Python.asdl:104-105@v3.15.0rc1": { "digest": "8781db4c9c043822", "first_line": "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift", "lines": 2 }, + "Parser/Python.asdl:105-105@v3.15.0rc1": { + "digest": "b3974786c24b0395", + "first_line": "| RShift | BitOr | BitXor | BitAnd | FloorDiv", + "lines": 1 + }, + "Parser/Python.asdl:107-107@v3.15.0rc1": { + "digest": "55374a939b510d53", + "first_line": "unaryop = Invert | Not | UAdd | USub", + "lines": 1 + }, + "Parser/Python.asdl:109-109@v3.15.0rc1": { + "digest": "f7677769589757c1", + "first_line": "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn", + "lines": 1 + }, + "Parser/Python.asdl:11-11@v3.15.0rc1": { + "digest": "2d2e1d84a1745db2", + "first_line": "stmt = FunctionDef(identifier name, arguments args,", + "lines": 1 + }, + "Parser/Python.asdl:111-111@v3.15.0rc1": { + "digest": "1c99246b3ea22eab", + "first_line": "comprehension = (expr target, expr iter, expr* ifs, int is_async)", + "lines": 1 + }, + "Parser/Python.asdl:113-113@v3.15.0rc1": { + "digest": "89bee9dfc89a347d", + "first_line": "excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)", + "lines": 1 + }, + "Parser/Python.asdl:116-116@v3.15.0rc1": { + "digest": "668a6d0337da6389", + "first_line": "arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs,", + "lines": 1 + }, + "Parser/Python.asdl:117-117@v3.15.0rc1": { + "digest": "574fc9c3b4c86106", + "first_line": "expr?* kw_defaults, arg? kwarg, expr* defaults)", + "lines": 1 + }, + "Parser/Python.asdl:119-119@v3.15.0rc1": { + "digest": "831486eb155f0f0d", + "first_line": "arg = (identifier arg, expr? annotation, string? type_comment)", + "lines": 1 + }, + "Parser/Python.asdl:123-123@v3.15.0rc1": { + "digest": "8e64105c2171a91b", + "first_line": "keyword = (identifier? arg, expr value)", + "lines": 1 + }, + "Parser/Python.asdl:127-127@v3.15.0rc1": { + "digest": "b1f77cf154ae3eca", + "first_line": "alias = (identifier name, identifier? asname)", + "lines": 1 + }, + "Parser/Python.asdl:130-130@v3.15.0rc1": { + "digest": "30f2a9dd2e71d243", + "first_line": "withitem = (expr context_expr, expr? optional_vars)", + "lines": 1 + }, + "Parser/Python.asdl:132-132@v3.15.0rc1": { + "digest": "89e74eaad33d9a34", + "first_line": "match_case = (pattern pattern, expr? guard, stmt* body)", + "lines": 1 + }, + "Parser/Python.asdl:134-134@v3.15.0rc1": { + "digest": "54dbc77feb5bb876", + "first_line": "pattern = MatchValue(expr value)", + "lines": 1 + }, + "Parser/Python.asdl:135-135@v3.15.0rc1": { + "digest": "280547049a21693c", + "first_line": "| MatchSingleton(constant value)", + "lines": 1 + }, + "Parser/Python.asdl:136-136@v3.15.0rc1": { + "digest": "ba84db8d159b75f3", + "first_line": "| MatchSequence(pattern* patterns)", + "lines": 1 + }, + "Parser/Python.asdl:137-137@v3.15.0rc1": { + "digest": "a471132c4c2a8187", + "first_line": "| MatchMapping(expr* keys, pattern* patterns, identifier? rest)", + "lines": 1 + }, + "Parser/Python.asdl:138-138@v3.15.0rc1": { + "digest": "023657bafb32a20b", + "first_line": "| MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)", + "lines": 1 + }, + "Parser/Python.asdl:14-14@v3.15.0rc1": { + "digest": "42169b3620851bc2", + "first_line": "| AsyncFunctionDef(identifier name, arguments args,", + "lines": 1 + }, + "Parser/Python.asdl:140-140@v3.15.0rc1": { + "digest": "bbb62caf097c0c18", + "first_line": "| MatchStar(identifier? name)", + "lines": 1 + }, + "Parser/Python.asdl:143-143@v3.15.0rc1": { + "digest": "8df0454f7c695351", + "first_line": "| MatchAs(pattern? pattern, identifier? name)", + "lines": 1 + }, + "Parser/Python.asdl:144-144@v3.15.0rc1": { + "digest": "8f909c91d0aad1ed", + "first_line": "| MatchOr(pattern* patterns)", + "lines": 1 + }, + "Parser/Python.asdl:148-148@v3.15.0rc1": { + "digest": "26d348017dcb0f6e", + "first_line": "type_ignore = TypeIgnore(int lineno, string tag)", + "lines": 1 + }, + "Parser/Python.asdl:150-150@v3.15.0rc1": { + "digest": "fe4e6835b5d4861e", + "first_line": "type_param = TypeVar(identifier name, expr? bound, expr? default_value)", + "lines": 1 + }, + "Parser/Python.asdl:151-151@v3.15.0rc1": { + "digest": "30a492aded8c0b71", + "first_line": "| ParamSpec(identifier name, expr? default_value)", + "lines": 1 + }, + "Parser/Python.asdl:152-152@v3.15.0rc1": { + "digest": "9ef13d5018b37948", + "first_line": "| TypeVarTuple(identifier name, expr? default_value)", + "lines": 1 + }, + "Parser/Python.asdl:18-18@v3.15.0rc1": { + "digest": "e08f7fa7d0a27d89", + "first_line": "| ClassDef(identifier name,", + "lines": 1 + }, + "Parser/Python.asdl:24-24@v3.15.0rc1": { + "digest": "3287588d5cd6bd44", + "first_line": "| Return(expr? value)", + "lines": 1 + }, + "Parser/Python.asdl:26-26@v3.15.0rc1": { + "digest": "982f4a4a4bd10382", + "first_line": "| Delete(expr* targets)", + "lines": 1 + }, + "Parser/Python.asdl:27-27@v3.15.0rc1": { + "digest": "9d2ab37ea4c84079", + "first_line": "| Assign(expr* targets, expr value, string? type_comment)", + "lines": 1 + }, + "Parser/Python.asdl:28-28@v3.15.0rc1": { + "digest": "86fbbbd15373ee4b", + "first_line": "| TypeAlias(expr name, type_param* type_params, expr value)", + "lines": 1 + }, + "Parser/Python.asdl:29-29@v3.15.0rc1": { + "digest": "fe694497f9da7e1e", + "first_line": "| AugAssign(expr target, operator op, expr value)", + "lines": 1 + }, + "Parser/Python.asdl:31-31@v3.15.0rc1": { + "digest": "a2a07393d3a8a650", + "first_line": "| AnnAssign(expr target, expr annotation, expr? value, int simple)", + "lines": 1 + }, + "Parser/Python.asdl:34-34@v3.15.0rc1": { + "digest": "4667ab9c45375f02", + "first_line": "| For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)", + "lines": 1 + }, + "Parser/Python.asdl:35-35@v3.15.0rc1": { + "digest": "ca53764ab36c71cf", + "first_line": "| AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)", + "lines": 1 + }, + "Parser/Python.asdl:36-36@v3.15.0rc1": { + "digest": "b0ed5bb3be3034b1", + "first_line": "| While(expr test, stmt* body, stmt* orelse)", + "lines": 1 + }, + "Parser/Python.asdl:37-37@v3.15.0rc1": { + "digest": "ccafd52ef652f5e9", + "first_line": "| If(expr test, stmt* body, stmt* orelse)", + "lines": 1 + }, + "Parser/Python.asdl:38-38@v3.15.0rc1": { + "digest": "43ff1688f7b94c15", + "first_line": "| With(withitem* items, stmt* body, string? type_comment)", + "lines": 1 + }, + "Parser/Python.asdl:39-39@v3.15.0rc1": { + "digest": "7b783047fe244938", + "first_line": "| AsyncWith(withitem* items, stmt* body, string? type_comment)", + "lines": 1 + }, + "Parser/Python.asdl:4-4@v3.15.0rc1": { + "digest": "b7d54fbd86358123", + "first_line": "module Python", + "lines": 1 + }, "Parser/Python.asdl:4-9@v3.15.0rc1": { "digest": "529a8ef072884926", "first_line": "module Python", "lines": 6 }, + "Parser/Python.asdl:41-41@v3.15.0rc1": { + "digest": "81a1e1fd6bf10cb0", + "first_line": "| Match(expr subject, match_case* cases)", + "lines": 1 + }, + "Parser/Python.asdl:43-43@v3.15.0rc1": { + "digest": "8f5eaa15a92f8387", + "first_line": "| Raise(expr? exc, expr? cause)", + "lines": 1 + }, + "Parser/Python.asdl:44-44@v3.15.0rc1": { + "digest": "ff752e0851f65bdf", + "first_line": "| Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)", + "lines": 1 + }, + "Parser/Python.asdl:45-45@v3.15.0rc1": { + "digest": "e68bf1b036cd1bcc", + "first_line": "| TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)", + "lines": 1 + }, + "Parser/Python.asdl:46-46@v3.15.0rc1": { + "digest": "2aa1a4a958c4a32b", + "first_line": "| Assert(expr test, expr? msg)", + "lines": 1 + }, + "Parser/Python.asdl:48-48@v3.15.0rc1": { + "digest": "c2cffda96af4d1c7", + "first_line": "| Import(alias* names, int? is_lazy)", + "lines": 1 + }, + "Parser/Python.asdl:49-49@v3.15.0rc1": { + "digest": "0f9c384abc1dac26", + "first_line": "| ImportFrom(identifier? module, alias* names, int? level, int? is_lazy)", + "lines": 1 + }, + "Parser/Python.asdl:51-51@v3.15.0rc1": { + "digest": "f8203827607389c5", + "first_line": "| Global(identifier* names)", + "lines": 1 + }, + "Parser/Python.asdl:52-52@v3.15.0rc1": { + "digest": "d9528e84d3609905", + "first_line": "| Nonlocal(identifier* names)", + "lines": 1 + }, + "Parser/Python.asdl:53-53@v3.15.0rc1": { + "digest": "5cda2b48284ff1ec", + "first_line": "| Expr(expr value)", + "lines": 1 + }, + "Parser/Python.asdl:54-54@v3.15.0rc1": { + "digest": "ff300189369537d5", + "first_line": "| Pass | Break | Continue", + "lines": 1 + }, + "Parser/Python.asdl:6-6@v3.15.0rc1": { + "digest": "6c5d3333f7fe832e", + "first_line": "mod = Module(stmt* body, type_ignore* type_ignores)", + "lines": 1 + }, + "Parser/Python.asdl:60-60@v3.15.0rc1": { + "digest": "2271e626a233d13d", + "first_line": "expr = BoolOp(boolop op, expr* values)", + "lines": 1 + }, + "Parser/Python.asdl:61-61@v3.15.0rc1": { + "digest": "5c3742565893966e", + "first_line": "| NamedExpr(expr target, expr value)", + "lines": 1 + }, "Parser/Python.asdl:62-62@v3.15.0rc1": { "digest": "13761ffe14f17084", "first_line": "| BinOp(expr left, operator op, expr right)", "lines": 1 }, + "Parser/Python.asdl:63-63@v3.15.0rc1": { + "digest": "0a94f4bd9da76ecb", + "first_line": "| UnaryOp(unaryop op, expr operand)", + "lines": 1 + }, + "Parser/Python.asdl:64-64@v3.15.0rc1": { + "digest": "497bb313234023c3", + "first_line": "| Lambda(arguments args, expr body)", + "lines": 1 + }, + "Parser/Python.asdl:65-65@v3.15.0rc1": { + "digest": "e85b11582667fdc8", + "first_line": "| IfExp(expr test, expr body, expr orelse)", + "lines": 1 + }, + "Parser/Python.asdl:66-66@v3.15.0rc1": { + "digest": "c4b8db3c4c0a26e1", + "first_line": "| Dict(expr?* keys, expr* values)", + "lines": 1 + }, + "Parser/Python.asdl:67-67@v3.15.0rc1": { + "digest": "f2bca3fa6926ec06", + "first_line": "| Set(expr* elts)", + "lines": 1 + }, + "Parser/Python.asdl:68-68@v3.15.0rc1": { + "digest": "e88481ea02381378", + "first_line": "| ListComp(expr elt, comprehension* generators)", + "lines": 1 + }, + "Parser/Python.asdl:69-69@v3.15.0rc1": { + "digest": "641b1d175fe6140a", + "first_line": "| SetComp(expr elt, comprehension* generators)", + "lines": 1 + }, + "Parser/Python.asdl:7-7@v3.15.0rc1": { + "digest": "a064c313c80b32f6", + "first_line": "| Interactive(stmt* body)", + "lines": 1 + }, + "Parser/Python.asdl:70-70@v3.15.0rc1": { + "digest": "41fa3cd5ed06f902", + "first_line": "| DictComp(expr key, expr? value, comprehension* generators)", + "lines": 1 + }, + "Parser/Python.asdl:71-71@v3.15.0rc1": { + "digest": "e2e448cab22be8b7", + "first_line": "| GeneratorExp(expr elt, comprehension* generators)", + "lines": 1 + }, + "Parser/Python.asdl:73-73@v3.15.0rc1": { + "digest": "b3455621cf61e482", + "first_line": "| Await(expr value)", + "lines": 1 + }, + "Parser/Python.asdl:74-74@v3.15.0rc1": { + "digest": "93cdd16da1689ab2", + "first_line": "| Yield(expr? value)", + "lines": 1 + }, + "Parser/Python.asdl:75-75@v3.15.0rc1": { + "digest": "0edd97658820d1c8", + "first_line": "| YieldFrom(expr value)", + "lines": 1 + }, + "Parser/Python.asdl:78-78@v3.15.0rc1": { + "digest": "393b9635a9eaa010", + "first_line": "| Compare(expr left, cmpop* ops, expr* comparators)", + "lines": 1 + }, + "Parser/Python.asdl:79-79@v3.15.0rc1": { + "digest": "7155b89886432191", + "first_line": "| Call(expr func, expr* args, keyword* keywords)", + "lines": 1 + }, + "Parser/Python.asdl:8-8@v3.15.0rc1": { + "digest": "05c4ca24ca535903", + "first_line": "| Expression(expr body)", + "lines": 1 + }, + "Parser/Python.asdl:80-80@v3.15.0rc1": { + "digest": "8505708806e72363", + "first_line": "| FormattedValue(expr value, int conversion, expr? format_spec)", + "lines": 1 + }, + "Parser/Python.asdl:81-81@v3.15.0rc1": { + "digest": "b207cc8e617caea7", + "first_line": "| Interpolation(expr value, constant str, int conversion, expr? format_spec)", + "lines": 1 + }, + "Parser/Python.asdl:82-82@v3.15.0rc1": { + "digest": "55e045248abeec74", + "first_line": "| JoinedStr(expr* values)", + "lines": 1 + }, + "Parser/Python.asdl:83-83@v3.15.0rc1": { + "digest": "d76bbdfddf879aab", + "first_line": "| TemplateStr(expr* values)", + "lines": 1 + }, + "Parser/Python.asdl:84-84@v3.15.0rc1": { + "digest": "c3e89c5a09c40e72", + "first_line": "| Constant(constant value, string? kind)", + "lines": 1 + }, + "Parser/Python.asdl:87-87@v3.15.0rc1": { + "digest": "6312c89d2edeeab0", + "first_line": "| Attribute(expr value, identifier attr, expr_context ctx)", + "lines": 1 + }, + "Parser/Python.asdl:88-88@v3.15.0rc1": { + "digest": "0eb83fc54aa0b958", + "first_line": "| Subscript(expr value, expr slice, expr_context ctx)", + "lines": 1 + }, + "Parser/Python.asdl:89-89@v3.15.0rc1": { + "digest": "ca58b46f31d8e655", + "first_line": "| Starred(expr value, expr_context ctx)", + "lines": 1 + }, + "Parser/Python.asdl:9-9@v3.15.0rc1": { + "digest": "88c7638923b29a61", + "first_line": "| FunctionType(expr* argtypes, expr returns)", + "lines": 1 + }, + "Parser/Python.asdl:90-90@v3.15.0rc1": { + "digest": "4dc7b53a6599bd0d", + "first_line": "| Name(identifier id, expr_context ctx)", + "lines": 1 + }, + "Parser/Python.asdl:91-91@v3.15.0rc1": { + "digest": "69e705cad286efe9", + "first_line": "| List(expr* elts, expr_context ctx)", + "lines": 1 + }, + "Parser/Python.asdl:92-92@v3.15.0rc1": { + "digest": "ca7d19d164137f8f", + "first_line": "| Tuple(expr* elts, expr_context ctx)", + "lines": 1 + }, + "Parser/Python.asdl:95-95@v3.15.0rc1": { + "digest": "1fa6e8e8abc36141", + "first_line": "| Slice(expr? lower, expr? upper, expr? step)", + "lines": 1 + }, + "Parser/asdl_c.py:1-2@v3.15.0rc1": { + "digest": "47980500884e7b9d", + "first_line": "#! /usr/bin/env python", + "lines": 2 + }, "Parser/asdl_c.py:1617-1619@v3.15.0rc1": { "digest": "6b8e76f86a28cf3b", "first_line": "static PyObject *", @@ -420,6 +885,31 @@ "first_line": "// File automatically generated by Parser/asdl_c.py.", "lines": 3 }, + "Python/Python-ast.c:18493-18499@v3.15.0rc1": { + "digest": "9b96334b5275d96a", + "first_line": "mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)", + "lines": 7 + }, + "Python/Python-ast.c:5266-5266@v3.15.0rc1": { + "digest": "ebf6045b3e8e4621", + "first_line": "ast_type_init(PyObject *self, PyObject *args, PyObject *kw)", + "lines": 1 + }, + "Python/Python-ast.c:5417-5446@v3.15.0rc1": { + "digest": "2f7d1ae8fa4ec594", + "first_line": "else if (_PyUnion_Check(type)) {", + "lines": 30 + }, + "Python/Python-ast.c:5448-5459@v3.15.0rc1": { + "digest": "a6069c402df403f2", + "first_line": "Py_ssize_t num_missing = PySet_GET_SIZE(missing_names);", + "lines": 12 + }, + "Python/Python-ast.c:7057-7089@v3.15.0rc1": { + "digest": "51259d0141d860cc", + "first_line": "_PyAST_FunctionDef(identifier name, arguments_ty args, asdl_stmt_seq * body,", + "lines": 33 + }, "Python/Python-ast.c:7767-7770@v3.15.0rc1": { "digest": "204a0cb5240c7286", "first_line": "_PyAST_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int", @@ -430,6 +920,26 @@ "first_line": "PyCodeObject *", "lines": 24 }, + "Python/ast.c:1049-1076@v3.15.0rc1": { + "digest": "a7d09b27fb03a92e", + "first_line": "_PyAST_Validate(mod_ty mod)", + "lines": 28 + }, + "Python/ast.c:13-18@v3.15.0rc1": { + "digest": "a4d9358c5bee15e5", + "first_line": "#define ENTER_RECURSIVE() \\", + "lines": 6 + }, + "Python/ast.c:704-711@v3.15.0rc1": { + "digest": "d8cd0372909c6fc3", + "first_line": "_validate_nonempty_seq(asdl_seq *seq, const char *what, const char *owner)", + "lines": 8 + }, + "Python/ast.c:722-726@v3.15.0rc1": { + "digest": "df067fd04b5b9e22", + "first_line": "validate_body(asdl_stmt_seq *body, const char *owner)", + "lines": 5 + }, "Python/ast_preprocess.c:370-383@v3.15.0rc1": { "digest": "ad12c7f3b5ca677d", "first_line": "static int", diff --git a/justfile b/justfile index f9f4e7f..ba390f4 100644 --- a/justfile +++ b/justfile @@ -55,9 +55,18 @@ citations: uv run refcheck verify # Check the blueprints have the shape somebody can implement from: the nine sections in -# order, the header block, the invariant numbering, and no fact deferred to a lesson. +# order, the header block, the invariant numbering, and no fact deferred to a lesson. The +# second line covers the ones with a source document under blueprints/sources: their +# generated sections have to still match what the grammar says today. blueprints: uv run bpcheck lint + uv run bpc check + +# Regenerate the compiled sections of a blueprint after editing its source document or +# moving the pin, then read the diff. Same deal as the diagrams and the notebooks: the +# output is committed because that is what GitHub renders, so something has to check it. +build-blueprints: + uv run bpc build # Confirm every committed diagram still matches the script that draws it. Same deal as the # notebooks below: the `.excalidraw` and the `.svg` are both generated and both committed, diff --git a/pyproject.toml b/pyproject.toml index 53524df..dcc9ec7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.14" license = "MIT" dependencies = [ + "bpc", "bpcheck", "nbbuild", "nbcheck", @@ -36,6 +37,7 @@ dev = [ [tool.uv.workspace] members = [ "pyxray", + "tools/bpc", "tools/bpcheck", "tools/nbbuild", "tools/nbcheck", @@ -46,6 +48,7 @@ members = [ ] [tool.uv.sources] +bpc = { workspace = true } bpcheck = { workspace = true } nbbuild = { workspace = true } nbcheck = { workspace = true } @@ -58,6 +61,7 @@ xraywidgets = { workspace = true } [tool.pytest.ini_options] testpaths = [ "pyxray/tests", + "tools/bpc/tests", "tools/bpcheck/tests", "tools/nbbuild/tests", "tools/nbcheck/tests", @@ -88,6 +92,7 @@ filterwarnings = [ [tool.ruff.lint.isort] known-first-party = [ + "bpc", "bpcheck", "nbbuild", "nbcheck", diff --git a/tools/bpc/README.md b/tools/bpc/README.md new file mode 100644 index 0000000..25f862f --- /dev/null +++ b/tools/bpc/README.md @@ -0,0 +1,67 @@ +# bpc + +The blueprint compiler. It writes the mechanical parts of a blueprint from CPython's own inputs, so that nobody types out 113 node kinds by hand and nobody has to notice when upstream adds a field. Run with `just build-blueprints`, and `just blueprints` checks that what is committed still matches. + +``` +uv run bpc list +uv run bpc build +uv run bpc check +``` + +## Why this exists + +A blueprint is two kinds of writing in one file. Sections 3, 4, 6, 7 and 9 are somebody's understanding of a subsystem, and there is no generating those. Sections 1, 2 and 5 of [BP-AST](../../blueprints/BP-AST.md) are a table of every type, every constructor and every field in `Parser/Python.asdl`, and that is transcription. Transcription is right on the day it is typed and wrong the first time the grammar changes, which for the AST is most releases. + +So the prose lives in `blueprints/sources/BP-AST.md` with a one line directive where each generated block belongs, and `bpc build` swaps each directive for the block and writes `blueprints/BP-AST.md`. Both files are committed. The output is what people read and what refcheck and bpcheck lint, and the source is what people edit. + +## The directives + +A directive is an HTML comment on a line of its own: + +``` + +``` + +There are four blocks. `overview` is section 1: the counts, and a row per type with a citation to the line it is declared on. `nodes` is section 2: a subsection per type, with a row per field of every constructor. `observable` is section 5: what the `ast` module shows of all this, with `_fields` and `_attributes` for every class. `conformance` is section 8: the table of claims and the tests that hold them up, with the counts filled in from the grammar. + +The expanded file keeps the boundary visible: + +``` + +...generated... + +``` + +That is what makes "no hand written content in a generated section" something a reader can check rather than something everybody has to remember. Editing between the markers is safe in the sense that nothing breaks immediately, and pointless in the sense that `bpc check` fails on the next run and the next build throws the edit away. + +## Where the line numbers come from + +CPython ships an ASDL parser at `Parser/asdl.py`, and `bpc` imports it from the pinned checkout rather than parsing the grammar itself. A second parser would be a second opinion about what the grammar means, and the reason to generate this material at all is that there should be one. + +What `asdl.py` does not give back is where anything was written. It parses to a tree of `Module`, `Type`, `Constructor` and `Field` with no line numbers anywhere. So `model.py` runs `asdl.py`'s own tokenizer a second time, which does carry line numbers, and walks the two in step. A definition is found by looking for a type name followed by `=`, which is what separates `arg` being declared on line 119 from `arg` being used as a field type on line 116. Anything the walk cannot follow raises rather than guessing, because a citation that points at the wrong line is worse than no citation. + +The result is that every citation in sections 1 and 2 points at a single line, and the name being cited is on it. If upstream moves a definition, the symbol is no longer where the citation says and `just citations` fails, instead of quietly pointing at whatever moved into that slot. + +## Layout + +``` +src/bpc/ + model.py the grammar as plain data, with line numbers attached + render.py grammar in, markdown out, no state and no file access + template.py swapping directives for blocks, and the errors when that goes wrong + cli.py build, check and list +tests/ + test_bpc_conformance.py the grammar against the running interpreter's ast module +``` + +`render.py` is pure, which is what makes the output deterministic: run it twice on the same pin and the bytes are identical, so a diff means the pin moved and nothing else. + +## The conformance tests + +`tests/test_bpc_conformance.py` is the half that checks the generated material is true rather than merely consistent. It reads the grammar from the pinned tree and compares it against the `ast` module of the interpreter running the tests: every type and constructor is a class, `_fields` and `_attributes` are the grammar's names in the grammar's order, and leaving a field out does what section 5 says. + +Those tests skip when the running interpreter's version does not match the pinned tag. A difference between `v3.15.0rc1` and whatever else is installed is a fact about the two versions, not a failure of the document, and failing the build for it would mean nobody could run the suite on anything but one interpreter. + +## What it does not do + +It does not lint. [bpcheck](../bpcheck) checks the structure of the output, and it lints `blueprints/BP-AST.md` without knowing it was generated, which is the point. It does not resolve citations, which [refcheck](../refcheck) does for every root in the repository. It does not repair drift on its own: `bpc check` reports and exits non zero, because a checker that silently fixes what it finds checks nothing, and the diff is the thing somebody is supposed to read before the pin moves. diff --git a/tools/bpc/pyproject.toml b/tools/bpc/pyproject.toml new file mode 100644 index 0000000..ddaf9f2 --- /dev/null +++ b/tools/bpc/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "bpc" +version = "0.1.0" +description = "The blueprint compiler, which generates blueprint sections from CPython's own inputs" +requires-python = ">=3.14" +license = "MIT" +dependencies = ["refcheck"] + +[project.scripts] +bpc = "bpc.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/bpc"] + +[tool.uv.sources] +refcheck = { workspace = true } diff --git a/tools/bpc/src/bpc/__init__.py b/tools/bpc/src/bpc/__init__.py new file mode 100644 index 0000000..c37bb6a --- /dev/null +++ b/tools/bpc/src/bpc/__init__.py @@ -0,0 +1,17 @@ +"""The blueprint compiler. + +The mechanical half of a specification is a transcription job, and transcription rots. A +list of every AST node and every field, typed by hand, is correct on the day it is written +and wrong the first time upstream adds a field. Nobody notices, because a reader who +trusted the list has no reason to go and check it. + +So the mechanical sections are generated from the same files CPython generates its own +front end from, and the generated output is committed next to the hand written prose. A +build that generates and a check that compares are two halves of the same rule: what is in +the repository is what the pinned tree says, or the build fails. +""" + +from .model import Constructor, Definition, Field, Grammar +from .template import Source, expand + +__all__ = ["Constructor", "Definition", "Field", "Grammar", "Source", "expand"] diff --git a/tools/bpc/src/bpc/cli.py b/tools/bpc/src/bpc/cli.py new file mode 100644 index 0000000..185ddbc --- /dev/null +++ b/tools/bpc/src/bpc/cli.py @@ -0,0 +1,114 @@ +"""The bpc command. + +`bpc build` expands every source document and writes the blueprint next to it. `bpc check` +expands the same documents and fails if what is committed has drifted, which is the half CI +runs. + +Two commands rather than one that repairs itself, for the same reason the notebooks and the +diagrams work this way: a checker that silently fixes what it finds checks nothing, and the +diff is the thing a person is supposed to read before the pin moves. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from refcheck.tree import TreeNotFound, find_tree + +from .model import GrammarError, grammar +from .template import OUTPUT, SOURCES, Source, TemplateError, expand, find + + +def _expanded(sources: list[Source], tree: Path) -> list[tuple[Source, str]]: + parsed = grammar(tree) + return [(source, expand(source, parsed)) for source in sources] + + +def command_build(args: argparse.Namespace) -> int: + sources, tree = _inputs(args) + if sources is None: + return 0 + for source, text in _expanded(sources, tree): + source.output.write_text(text, encoding="utf-8") + print(f"wrote {source.output}") + print(f"{len(sources)} blueprint(s) generated") + return 0 + + +def command_check(args: argparse.Namespace) -> int: + sources, tree = _inputs(args) + if sources is None: + return 0 + problems = [] + for source, text in _expanded(sources, tree): + if not source.output.exists(): + problems.append(f"{source.output} has not been built, run `just build-blueprints`") + elif source.output.read_text(encoding="utf-8") != text: + problems.append( + f"{source.output} no longer matches {source.path}, " + "run `just build-blueprints` and read the diff" + ) + for problem in problems: + print(problem, file=sys.stderr) + print(f"{len(sources)} blueprint(s): {len(problems)} out of date") + return 1 if problems else 0 + + +def command_list(args: argparse.Namespace) -> int: + sources, _ = _inputs(args, need_tree=False) + if sources is None: + return 0 + for source in sources: + print(f"{source.path} -> {source.output}: {', '.join(source.blocks())}") + return 0 + + +def _inputs( + args: argparse.Namespace, *, need_tree: bool = True +) -> tuple[list[Source] | None, Path]: + sources = find(Path(args.sources)) + if not sources: + print(f"no source documents under {args.sources}", file=sys.stderr) + return None, Path() + if not need_tree: + return sources, Path() + return sources, find_tree(getattr(args, "tree", None)) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="bpc", + description="Generate the mechanical sections of a blueprint from CPython's own inputs", + ) + parser.add_argument("--sources", default=str(SOURCES), help=f"defaults to {SOURCES}") + parser.add_argument("--tree", default=None, help="a CPython checkout at the pinned tag") + sub = parser.add_subparsers(dest="command", required=True) + + build = sub.add_parser("build", help=f"expand every source document into {OUTPUT}") + build.set_defaults(func=command_build) + + check = sub.add_parser("check", help="fail if a committed blueprint has drifted") + check.set_defaults(func=command_check) + + listing = sub.add_parser("list", help="show the source documents and the blocks they use") + listing.set_defaults(func=command_list) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return args.func(args) + except TreeNotFound as error: + print(error, file=sys.stderr) + return 2 + except (GrammarError, TemplateError) as error: + print(error, file=sys.stderr) + return 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tools/bpc/src/bpc/model.py b/tools/bpc/src/bpc/model.py new file mode 100644 index 0000000..2d4850c --- /dev/null +++ b/tools/bpc/src/bpc/model.py @@ -0,0 +1,310 @@ +"""The grammar, as plain data with line numbers attached. + +CPython's own `Parser/asdl.py` does the parsing. Writing a second ASDL parser here would +be writing a second opinion about what `Parser/Python.asdl` means, and the whole point of +generating this material is that there is only one opinion in the repository. + +What `asdl.py` does not give back is where anything was written. It parses to a tree of +`Module`, `Type`, `Constructor` and `Field` with no line numbers on any of them, and a +specification that cannot point at the line it came from is a specification a reader has +to take on trust. So this module runs `asdl.py`'s own tokenizer a second time, which does +carry line numbers, and walks the two in step. The walk asserts that the names line up in +order, so a change in upstream that this code cannot follow stops the build rather than +quietly producing citations that point at the wrong lines. +""" + +from __future__ import annotations + +import importlib.util +import sys +from dataclasses import dataclass +from functools import cache +from pathlib import Path +from types import ModuleType + +#: Where the two files live inside a CPython checkout. +ASDL_MODULE = Path("Parser") / "asdl.py" +GRAMMAR_FILE = Path("Parser") / "Python.asdl" + +#: The four types ASDL has built in, which no definition in the file declares. Everything +#: else named as a field type is a definition in the same file, and `asdl.py` checks that. +BUILTIN_TYPES = frozenset({"identifier", "int", "string", "constant"}) + + +class GrammarError(RuntimeError): + """The grammar could not be read, or could not be lined up with its own source.""" + + +@dataclass(frozen=True) +class Field: + """One field of one node, in the order it is written.""" + + type: str + name: str + optional: bool + sequence: bool + marks: str = "" + + @property + def builtin(self) -> bool: + """Whether the field's type is one of ASDL's four, rather than a node type.""" + return self.type in BUILTIN_TYPES + + @property + def elements_optional(self) -> bool: + """Whether this is a sequence whose slots are allowed to be empty. + + There is exactly one of these in the grammar and it is easy to miss, because + `asdl.py` keeps only the last quantifier in `seq` and `opt`, so `expr?* kw_defaults` + arrives looking like an ordinary sequence. The `?` is still in `quantifiers` and it + is load bearing: `kw_defaults` holds one slot per keyword only argument, and the + slot is `None` for an argument that has no default. + """ + return self.sequence and "?" in self.marks + + @property + def kind(self) -> str: + """How many values the field holds, in a few words. + + A port needs this and needs it separately from the type. A sequence field is + always present and may be empty, an optional field may be absent altogether, and + those two are different in a way that `expr* body` and `expr? returns` hide from + anybody reading quickly. + """ + if self.elements_optional: + return "sequence of optional" + if self.sequence: + return "sequence" + return "optional" if self.optional else "required" + + @property + def notation(self) -> str: + """The field as ASDL writes it, `expr? returns` and the like.""" + return f"{self.type}{self.marks} {self.name}" + + +@dataclass(frozen=True) +class Constructor: + """One concrete node kind, which is one alternative of a sum.""" + + name: str + fields: tuple[Field, ...] + line: int + + @property + def signature(self) -> str: + """The constructor as ASDL writes it, arguments and all.""" + inner = ", ".join(field.notation for field in self.fields) + return f"{self.name}({inner})" if self.fields else self.name + + +@dataclass(frozen=True) +class Definition: + """One named type in the grammar, either a sum of constructors or a single product.""" + + name: str + constructors: tuple[Constructor, ...] + fields: tuple[Field, ...] + attributes: tuple[Field, ...] + line: int + end_line: int + + @property + def sum(self) -> bool: + """Whether this is a choice between constructors rather than a single shape.""" + return bool(self.constructors) + + @property + def kind(self) -> str: + return "sum" if self.sum else "product" + + +@dataclass(frozen=True) +class Grammar: + """Everything in `Parser/Python.asdl`, in the order the file writes it.""" + + name: str + definitions: tuple[Definition, ...] + line: int + path: str = str(GRAMMAR_FILE) + + def definition(self, name: str) -> Definition: + for one in self.definitions: + if one.name == name: + return one + raise KeyError(name) + + @property + def node_count(self) -> int: + """How many concrete node kinds there are, which is what a port has to build.""" + return sum(max(len(one.constructors), 1) for one in self.definitions) + + +def load_asdl(tree: Path) -> ModuleType: + """Import CPython's `Parser/asdl.py` from the pinned checkout. + + By path rather than by adding `Parser` to `sys.path`, because that directory also holds + `pegen` and a handful of other modules with names general enough to shadow something. + The module is cached under a name that cannot collide with anything installed. + """ + path = tree / ASDL_MODULE + if not path.is_file(): + raise GrammarError(f"no {ASDL_MODULE} in {tree}, so there is nothing to compile from") + name = "bpc._cpython_asdl" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise GrammarError(f"{path} could not be loaded as a module") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +#: How `asdl.py` names its two quantifiers, and what the grammar file writes for each. +QUANTIFIERS = {"SEQUENCE": "*", "OPTIONAL": "?"} + + +def _marks(one: object) -> str: + """The quantifiers after a field's type, in the order the grammar writes them. + + `seq` and `opt` would nearly do, but they are set from the last quantifier only, so + they turn `expr?* kw_defaults` into `expr* kw_defaults` and lose the fact that the + slots of that sequence may be empty. The full list is on the field, so use it. + """ + quantifiers = getattr(one, "quantifiers", None) + if not quantifiers: + return "*" if one.seq else "?" if one.opt else "" + return "".join(QUANTIFIERS[str(mark.name)] for mark in quantifiers) + + +def _fields(raw: object) -> tuple[Field, ...]: + return tuple( + Field( + type=one.type, + name=one.name, + optional=bool(one.opt), + sequence=bool(one.seq), + marks=_marks(one), + ) + for one in raw + ) + + +class _Lines: + """Where each name in the grammar was written, from `asdl.py`'s own tokenizer. + + The tokenizer yields `(kind, value, lineno)` in file order, which is the same order + the parsed tree is in, so lining them up is a forward scan with no lookahead. Anything + that does not line up raises, because a citation pointing at the wrong line is worse + than no citation. + """ + + def __init__(self, asdl: ModuleType, text: str) -> None: + self._kinds = asdl.TokenKind + self._tokens = list(asdl.tokenize_asdl(text)) + self._at = 0 + + @property + def last_line(self) -> int: + return self._tokens[-1].lineno if self._tokens else 0 + + def module(self, name: str) -> int: + """The line the `module NAME {` header is on.""" + return self.constructor(name) + + def definition(self, name: str) -> int: + """The line a definition's name is on. + + A type name followed by `=` and nothing else. The `=` is what makes this a + definition rather than a use: `arg` is a definition on one line and the type of + three fields of `arguments` several lines earlier, and taking the first `arg` in + the file would put every citation for it on the wrong line. + """ + index = self._find_definition(name, self._at) + if index is None: + raise self._lost(name) + self._at = index + 1 + return self._tokens[index].lineno + + def constructor(self, name: str) -> int: + """The line a constructor's name is on. + + No `=` check here, because a constructor name is capitalised and a capitalised + name in this grammar is only ever a constructor being declared. + """ + while self._at < len(self._tokens): + token = self._tokens[self._at] + self._at += 1 + if token.kind == self._kinds.ConstructorId and token.value == name: + return token.lineno + raise self._lost(name) + + def peek_definition(self, name: str) -> int | None: + """The line the next definition starts on, without consuming anything.""" + index = self._find_definition(name, self._at) + return None if index is None else self._tokens[index].lineno + + def _find_definition(self, name: str, start: int) -> int | None: + for index in range(start, len(self._tokens) - 1): + token = self._tokens[index] + if token.kind != self._kinds.TypeId or token.value != name: + continue + if self._tokens[index + 1].kind == self._kinds.Equals: + return index + return None + + def _lost(self, name: str) -> GrammarError: + return GrammarError( + f"the tokenizer never reached {name!r}, so the grammar and its own source " + "have stopped lining up and the line numbers cannot be trusted" + ) + + +def parse(tree: Path) -> Grammar: + """Read `Parser/Python.asdl` from the pinned checkout, with line numbers attached.""" + asdl = load_asdl(tree) + path = tree / GRAMMAR_FILE + if not path.is_file(): + raise GrammarError(f"no {GRAMMAR_FILE} in {tree}, so there is nothing to compile from") + text = path.read_text(encoding="utf-8") + module = asdl.parse(str(path)) + + lines = _Lines(asdl, text) + module_line = lines.module(module.name) + + names = [one.name for one in module.dfns] + definitions: list[Definition] = [] + for index, dfn in enumerate(module.dfns): + start = lines.definition(dfn.name) + value = dfn.value + constructors = tuple( + Constructor( + name=one.name, + fields=_fields(one.fields), + line=lines.constructor(one.name), + ) + for one in getattr(value, "types", []) + ) + following = names[index + 1] if index + 1 < len(names) else None + after = lines.peek_definition(following) if following else None + end = (after - 1) if after else lines.last_line + definitions.append( + Definition( + name=dfn.name, + constructors=constructors, + fields=_fields(getattr(value, "fields", [])), + attributes=_fields(getattr(value, "attributes", [])), + line=start, + end_line=max(end, start), + ) + ) + + return Grammar(name=module.name, definitions=tuple(definitions), line=module_line) + + +@cache +def grammar(tree: Path) -> Grammar: + """`parse`, remembered, because a build reads the grammar once per section.""" + return parse(tree) diff --git a/tools/bpc/src/bpc/render.py b/tools/bpc/src/bpc/render.py new file mode 100644 index 0000000..9266f78 --- /dev/null +++ b/tools/bpc/src/bpc/render.py @@ -0,0 +1,293 @@ +"""Turning the grammar into the parts of a blueprint that nobody should be typing. + +Every function here takes the grammar and returns markdown. There is no state, no file +access and no formatting that depends on anything but the grammar, which is what makes the +output deterministic: run it twice on the same pin and the bytes are identical, so a diff +against the previous run means the pin moved and nothing else. + +The tables are wide rather than clever. A specification is read by somebody who is looking +for one field of one node, so every row repeats the node name instead of relying on a +reader keeping their finger on the last one, and every row can be found with grep. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator + +from refcheck.citation import find_all +from refcheck.tree import PINNED_TAG + +from .model import Constructor, Definition, Grammar + +#: The four location attributes CPython puts on the node types that can be pointed at in +#: an error message. Named here only so the generated prose can say how many there are. +LOCATION_ATTRIBUTES = ("lineno", "col_offset", "end_lineno", "end_col_offset") + + +def citation(grammar: Grammar, line: int, symbol: str) -> str: + """One citation into the pinned grammar file, as a code span. + + A single line rather than a range on purpose. The name being cited is on that line, so + the citation is self checking: if upstream moves the definition, the symbol is no + longer where the citation says it is and `just citations` fails instead of quietly + pointing at whatever moved into that slot. + """ + return f"`{grammar.path}:{line}@{PINNED_TAG}#{symbol}`" + + +def table(headings: Iterable[str], rows: Iterable[Iterable[str]]) -> list[str]: + """A markdown table, with the header separator markdown insists on.""" + names = list(headings) + lines = ["| " + " | ".join(names) + " |", "|" + "|".join(["---"] * len(names)) + "|"] + lines.extend("| " + " | ".join(str(cell) for cell in row) + " |" for row in rows) + return lines + + +def overview(grammar: Grammar) -> str: + """Section 1: what the grammar contains, counted.""" + sums = sum(1 for one in grammar.definitions if one.sum) + fields = sum(len(one.fields) for one in grammar.definitions) + fields += sum(len(two.fields) for one in grammar.definitions for two in one.constructors) + carrying = [one.name for one in grammar.definitions if one.attributes] + + lines = [ + f"The grammar declares {len(grammar.definitions)} types, {sums} of them a choice " + f"between constructors and {len(grammar.definitions) - sums} of them a single fixed " + f"shape. Between them they describe {grammar.node_count} concrete node kinds with " + f"{fields} fields, and {len(carrying)} of the types carry source location attributes " + "on every node.", + "", + f"The module header is at {citation(grammar, grammar.line, grammar.name)}.", + "", + ] + rows = [] + for index, one in enumerate(grammar.definitions, start=1): + rows.append( + [ + str(index), + f"`{one.name}`", + one.kind, + str(len(one.constructors)) if one.sum else "", + str(_field_count(one)), + str(len(one.attributes)), + citation(grammar, one.line, one.name), + ] + ) + lines.extend( + table( + ["#", "Type", "Kind", "Constructors", "Fields", "Attributes", "Declared at"], + rows, + ) + ) + return "\n".join(lines) + + +def nodes(grammar: Grammar) -> str: + """Section 2: every type, every constructor, every field, in declaration order.""" + lines: list[str] = [] + for index, one in enumerate(grammar.definitions, start=1): + if lines: + lines.append("") + lines.extend(_definition(grammar, one, f"2.{index}")) + return "\n".join(lines) + + +def observable(grammar: Grammar) -> str: + """Section 5: what a Python program can see of all this through the `ast` module.""" + lines = [ + f"The whole grammar is visible from Python. Each of the {grammar.node_count} node " + f"kinds below is a class in the `ast` module with the same name, each of the " + f"{len(grammar.definitions)} types is a class those inherit from, and the field " + "order in the grammar is the order those classes take positional arguments in. A " + "reimplementation that renames a field or reorders two of them is detectable by " + "any program that builds a tree by hand or reads one back.", + "", + "The three field kinds are three different things to leave out. A required field " + "has to be passed, and building the node without it raises `TypeError` naming the " + "field. An optional field left out is `None`. A sequence field left out is a new " + "empty list, so `body` is `[]` rather than missing. There is one field type that " + "breaks the pattern: a field of type `expr_context` left out is the `Load` " + "singleton, because nearly every expression in a tree is being read rather than " + "written to.", + "", + f"The {len(LOCATION_ATTRIBUTES)} location attributes are separate from the fields. " + "They are listed in `_attributes` rather than `_fields`, and none of them is ever " + "required by the constructor, so a node can always be built without them. The two " + "declared optional default to `None` like any other optional. The two declared " + "required have no value at all, and reading one raises `AttributeError` rather than " + "returning `None`. Nothing complains until `compile` sees the tree, which is where " + "a missing line number becomes `TypeError` and where a port has to put the same " + "check.", + "", + ] + rows = [] + for one in grammar.definitions: + rows.append( + [ + f"`{one.name}`", + "abstract" if one.sum else "concrete", + _tuple(field.name for field in one.fields), + _tuple(field.name for field in one.attributes), + ] + ) + for two in one.constructors: + rows.append( + [ + f"`{two.name}`", + f"`{one.name}`", + _tuple(field.name for field in two.fields), + _tuple(field.name for field in one.attributes), + ] + ) + lines.extend(table(["Class", "Base", "`_fields`", "`_attributes`"], rows)) + return "\n".join(lines) + + +def conformance(grammar: Grammar) -> str: + """Section 8: what holds the two sections above up, and how much of them it covers.""" + fields = sum(len(one.fields) for one in grammar.definitions) + fields += sum(len(two.fields) for one in grammar.definitions for two in one.constructors) + carrying = sum(1 for one in grammar.definitions if one.attributes) + citations = len(find_all(overview(grammar))) + len(find_all(nodes(grammar))) + + lines = [ + "Sections 1, 2 and 5 are generated from the grammar file, so the way they go wrong " + "is not a typo. They go wrong when the running interpreter and the pinned grammar " + "have stopped agreeing, which is what the checks below are for. Each one reads the " + "grammar from the pinned tree and compares it against the `ast` module of the " + "interpreter running the test.", + "", + ] + rows = [ + [ + "Every type in section 1 is a class in `ast`", + "`test_every_type_in_the_grammar_is_a_class_in_ast`", + f"{len(grammar.definitions)} types", + ], + [ + "Every constructor in section 2 is a class in `ast`", + "`test_every_constructor_in_the_grammar_is_a_class_in_ast`", + f"{grammar.node_count} node kinds", + ], + [ + "`_fields` is the grammar's field names, in the grammar's order", + "`test_the_field_order_is_the_order_the_grammar_declares`", + f"{fields} fields", + ], + [ + "`_attributes` is the grammar's attributes, in the grammar's order", + "`test_the_attributes_are_the_ones_the_grammar_declares`", + f"{carrying} types carry attributes", + ], + [ + "Leaving a field out does what section 5 says it does", + "`test_the_defaults_are_the_ones_section_5_describes`", + "the three field kinds", + ], + [ + "Every citation generated into sections 1 and 2 resolves against the pinned tree", + "`just citations`", + f"{citations} citations", + ], + ] + lines.extend(table(["Claim", "Held up by", "Covers"], rows)) + lines.extend( + [ + "", + "The first five run under `just test` and live in `tools/bpc/tests/" + "test_bpc_conformance.py`. They are skipped on an interpreter whose version does " + f"not match the pinned tree, because a difference between {PINNED_TAG} and some " + "other version is a fact about the two versions rather than a failure of this " + "document.", + ] + ) + return "\n".join(lines) + + +def _definition(grammar: Grammar, one: Definition, number: str) -> list[str]: + """One type: its heading, what it is, and a row per field.""" + lines = [f"### {number} `{one.name}`", ""] + where = citation(grammar, one.line, one.name) + if one.sum: + lines.append(f"A choice between {len(one.constructors)} constructors, declared at {where}.") + else: + lines.append( + f"A single shape with {len(one.fields)} fields, declared at {where}. There is " + "nothing to switch on: every value of this type has exactly these fields." + ) + lines.append("") + + if one.attributes: + lines.append( + f"Every `{one.name}` node also carries {len(one.attributes)} attributes, which " + "are not fields and are not part of the constructor's positional arguments." + ) + lines.append("") + lines.extend( + table( + ["Attribute", "Type", "Holds"], + [[f"`{field.name}`", f"`{field.type}`", field.kind] for field in one.attributes], + ) + ) + lines.append("") + + if one.sum: + lines.extend( + table( + ["Node", "Order", "Field", "Type", "Holds", "Declared at"], + _sum_rows(grammar, one), + ) + ) + else: + lines.extend(table(["Order", "Field", "Type", "Holds"], _product_rows(one))) + return lines + + +def _sum_rows(grammar: Grammar, one: Definition) -> Iterator[list[str]]: + for two in one.constructors: + where = citation(grammar, two.line, two.name) + if not two.fields: + yield [f"`{two.name}`", "", "no fields", "", "", where] + continue + for order, field in enumerate(two.fields, start=1): + yield [ + f"`{two.name}`", + str(order), + f"`{field.name}`", + f"`{field.type}`", + field.kind, + where if order == 1 else "", + ] + + +def _product_rows(one: Definition) -> Iterator[list[str]]: + for order, field in enumerate(one.fields, start=1): + yield [str(order), f"`{field.name}`", f"`{field.type}`", field.kind] + + +def _field_count(one: Definition) -> int: + if not one.sum: + return len(one.fields) + return sum(len(two.fields) for two in one.constructors) + + +def _tuple(names: Iterable[str]) -> str: + """A Python tuple of strings, written the way `_fields` prints.""" + inside = ", ".join(f"'{name}'" for name in names) + if not inside: + return "`()`" + return f"`({inside},)`" if inside.count(",") == 0 else f"`({inside})`" + + +def signature(two: Constructor) -> str: + """A constructor as the grammar writes it, for anywhere a table is too much.""" + return two.signature + + +#: The blocks a source document can ask for, by the name it writes in its directive. +BLOCKS = { + "overview": overview, + "nodes": nodes, + "observable": observable, + "conformance": conformance, +} diff --git a/tools/bpc/src/bpc/template.py b/tools/bpc/src/bpc/template.py new file mode 100644 index 0000000..8e0e4b0 --- /dev/null +++ b/tools/bpc/src/bpc/template.py @@ -0,0 +1,119 @@ +"""Expanding a source document into the blueprint that gets committed. + +A blueprint is part specification and part transcription. The specification part is written +by somebody who understands the subsystem and cannot be generated. The transcription part +is a table of every node and every field, which can be generated and therefore should be, +because a hand typed one is right the day it is written and wrong the first time upstream +adds a field. + +So the source document holds the prose with a one line directive wherever a generated block +belongs, and this module swaps each directive for the block. The markers left behind in the +output are HTML comments, which are invisible on GitHub and obvious in an editor, and they +are what makes "no hand written content in a generated section" a thing anybody can see +rather than a thing everybody has to remember. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +from .model import Grammar +from .render import BLOCKS + +#: `` on a line of its own, which is what a source document writes. +DIRECTIVE = re.compile(r"^$") + +#: What the expanded document has instead, so the boundary survives into the output. +BEGIN = "" +END = "" + +#: Where the source documents live, and where their output goes. +SOURCES = Path("blueprints") / "sources" +OUTPUT = Path("blueprints") + + +class TemplateError(RuntimeError): + """The source document asked for something that cannot be generated.""" + + +@dataclass(frozen=True) +class Source: + """One source document, and the file it produces.""" + + path: Path + + @property + def name(self) -> str: + """`BP-AST`, which is the name of both the source and the output.""" + return self.path.stem + + @property + def output(self) -> Path: + """Where the expanded document belongs, which is one directory up.""" + return self.path.parent.parent / self.path.name + + def text(self) -> str: + return self.path.read_text(encoding="utf-8") + + def blocks(self) -> list[str]: + """The generated blocks this document asks for, in the order it asks for them.""" + found = [] + for line in self.text().splitlines(): + match = DIRECTIVE.match(line) + if match is not None: + found.append(match.group(1)) + return found + + +def expand(source: Source, grammar: Grammar) -> str: + """The finished document: the prose, with every directive replaced by its block. + + Every generated block is checked as it goes in. An empty one means the grammar changed + shape in a way the renderer did not follow, and shipping a section with a heading and + nothing under it is how a specification comes to have a hole in it that reads like a + subsystem with nothing to say. + """ + seen: set[str] = set() + out: list[str] = [] + for number, line in enumerate(source.text().splitlines(), start=1): + match = DIRECTIVE.match(line) + if match is None: + out.append(line) + continue + name = match.group(1) + if name not in BLOCKS: + known = ", ".join(sorted(BLOCKS)) + raise TemplateError( + f"{source.path}:{number}: there is no block called {name!r}, only {known}" + ) + if name in seen: + raise TemplateError( + f"{source.path}:{number}: the {name!r} block is asked for twice, and a " + "specification that states the same table in two places has two of them to " + "keep in step" + ) + seen.add(name) + body = BLOCKS[name](grammar) + if not body.strip(): + raise TemplateError( + f"{source.path}:{number}: the {name!r} block came out empty, which means the " + "grammar no longer has what the renderer went looking for" + ) + out.append(BEGIN.format(name=name)) + out.append(body) + out.append(END.format(name=name)) + if not seen: + raise TemplateError( + f"{source.path}: no `` directives, so there is nothing here " + "that bpc can generate and the file does not need to be a source document" + ) + return "\n".join(out).rstrip("\n") + "\n" + + +def find(root: Path = SOURCES) -> list[Source]: + """Every source document under `blueprints/sources`, sorted.""" + if not root.is_dir(): + return [] + return [Source(path) for path in sorted(root.glob("BP-*.md"))] diff --git a/tools/bpc/tests/conftest.py b/tools/bpc/tests/conftest.py new file mode 100644 index 0000000..7deb844 --- /dev/null +++ b/tools/bpc/tests/conftest.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import pytest + +from bpc.model import grammar +from refcheck.tree import PINNED_TAG, TreeNotFound, find_tree + + +@pytest.fixture(scope="session") +def tree(): + """The pinned CPython checkout, or a skip if this machine does not have one. + + Same bargain as refcheck's: somebody changing the renderer should not need 200 MB of + CPython on disk to run the tests that cover it, and CI always has the tree, so + everything here runs somewhere. + """ + try: + return find_tree() + except TreeNotFound as error: + pytest.skip(str(error)) + + +@pytest.fixture(scope="session") +def asdl(tree): + """The parsed grammar, read once for the whole session.""" + return grammar(tree) + + +@pytest.fixture(scope="session") +def pinned_interpreter(): + """A skip unless the interpreter running the tests is the pinned version. + + The conformance tests compare the pinned grammar against the `ast` module of whatever + is running them. On a different version a difference is a fact about the two versions + rather than a failure of the document, so there is nothing to report and nothing to fix. + """ + import sys + + running = ".".join(str(part) for part in sys.version_info[:2]) + wanted = PINNED_TAG.removeprefix("v").split("rc")[0].rsplit(".", 1)[0] + if running != wanted: + pytest.skip(f"running {running}, the pin is {wanted}") diff --git a/tools/bpc/tests/test_bpc_cli.py b/tools/bpc/tests/test_bpc_cli.py new file mode 100644 index 0000000..2f6695b --- /dev/null +++ b/tools/bpc/tests/test_bpc_cli.py @@ -0,0 +1,124 @@ +"""The three commands, and the exit codes CI reads. + +`check` returning 1 rather than repairing what it finds is the whole design, so most of +what is here is about that: it has to notice a missing output, notice a changed one, and +say which file and what to run. +""" + +from __future__ import annotations + +import pytest + +from bpc.cli import main + + +@pytest.fixture +def sources(tmp_path): + """A source tree with one small document, laid out the way the repository is.""" + root = tmp_path / "blueprints" / "sources" + root.mkdir(parents=True) + (root / "BP-TOY.md").write_text( + "# BP-TOY: a toy\n\nbefore\n\n\n\nafter\n", encoding="utf-8" + ) + return root + + +def run(sources, *args, tree=None): + argv = ["--sources", str(sources)] + if tree is not None: + argv += ["--tree", str(tree)] + return main([*argv, *args]) + + +def test_list_shows_the_output_path_and_the_blocks(sources, capsys): + assert run(sources, "list") == 0 + out = capsys.readouterr().out + assert "BP-TOY.md -> " in out + assert out.rstrip().endswith(": overview") + + +def test_list_does_not_need_a_checkout(sources, capsys, monkeypatch): + """Listing is about the documents, so it works on a machine with no CPython on it.""" + monkeypatch.delenv("CPYTHON_SRC", raising=False) + assert run(sources, "list") == 0 + + +def test_build_writes_the_output_one_directory_up(sources, tree, capsys): + assert run(sources, "build", tree=tree) == 0 + written = sources.parent / "BP-TOY.md" + assert written.exists() + assert "" in written.read_text(encoding="utf-8") + assert "1 blueprint(s) generated" in capsys.readouterr().out + + +def test_build_twice_writes_the_same_bytes(sources, tree): + run(sources, "build", tree=tree) + first = (sources.parent / "BP-TOY.md").read_bytes() + run(sources, "build", tree=tree) + assert (sources.parent / "BP-TOY.md").read_bytes() == first + + +def test_check_passes_on_what_build_just_wrote(sources, tree, capsys): + run(sources, "build", tree=tree) + assert run(sources, "check", tree=tree) == 0 + assert "0 out of date" in capsys.readouterr().out + + +def test_check_fails_when_the_output_was_never_built(sources, tree, capsys): + assert run(sources, "check", tree=tree) == 1 + assert "has not been built" in capsys.readouterr().err + + +def test_check_fails_when_somebody_edited_the_generated_part(sources, tree, capsys): + run(sources, "build", tree=tree) + written = sources.parent / "BP-TOY.md" + written.write_text( + written.read_text(encoding="utf-8").replace("before", "edited"), encoding="utf-8" + ) + assert run(sources, "check", tree=tree) == 1 + err = capsys.readouterr().err + assert "no longer matches" in err + assert "just build-blueprints" in err + + +def test_check_does_not_repair_what_it_finds(sources, tree): + run(sources, "build", tree=tree) + written = sources.parent / "BP-TOY.md" + written.write_text("broken\n", encoding="utf-8") + run(sources, "check", tree=tree) + assert written.read_text(encoding="utf-8") == "broken\n" + + +def test_no_source_documents_says_where_it_looked(tmp_path, capsys): + empty = tmp_path / "sources" + empty.mkdir() + assert run(empty, "build") == 0 + assert "no source documents under" in capsys.readouterr().err + + +def test_a_bad_directive_exits_one_and_explains(sources, tree, capsys): + (sources / "BP-TOY.md").write_text("\n", encoding="utf-8") + assert run(sources, "build", tree=tree) == 1 + assert "there is no block called 'nope'" in capsys.readouterr().err + + +def test_no_checkout_anywhere_exits_two(sources, capsys, monkeypatch): + """Two rather than one, so CI can tell a missing checkout from a real failure. + + `find_tree` is replaced rather than pointed at an empty directory, because it falls + back to `CPYTHON_SRC` and then to `vendor/cpython`, and on a machine that has either + of those a bad `--tree` quietly succeeds. + """ + from refcheck.tree import TreeNotFound + + def missing(_): + raise TreeNotFound("no CPython checkout found") + + monkeypatch.setattr("bpc.cli.find_tree", missing) + assert run(sources, "build") == 2 + assert "no CPython checkout found" in capsys.readouterr().err + + +def test_a_command_is_required(capsys): + with pytest.raises(SystemExit): + main([]) diff --git a/tools/bpc/tests/test_bpc_conformance.py b/tools/bpc/tests/test_bpc_conformance.py new file mode 100644 index 0000000..79ac03b --- /dev/null +++ b/tools/bpc/tests/test_bpc_conformance.py @@ -0,0 +1,190 @@ +"""The generated sections of BP-AST, checked against the interpreter running the tests. + +Sections 1, 2 and 5 of the blueprint are transcribed from `Parser/Python.asdl` by machine, +so they cannot contain a typo. What they can contain is a claim that was true of the pinned +grammar and is not true of the `ast` module in front of the reader, and that is what these +tests are for. Every one of them reads the grammar from the pinned tree and compares it +against `ast` directly. + +Section 8 of the blueprint names these tests by their function names, so renaming one here +means changing `render.py` and rebuilding. That is intentional. A conformance section that +points at a test which does not exist is the failure mode this whole arrangement is meant +to avoid. +""" + +from __future__ import annotations + +import ast + +import pytest + +from bpc.model import BUILTIN_TYPES + +pytestmark = pytest.mark.usefixtures("pinned_interpreter") + + +def test_every_type_in_the_grammar_is_a_class_in_ast(asdl): + """Section 1's first column, one name at a time.""" + for one in asdl.definitions: + node = getattr(ast, one.name, None) + assert node is not None, f"the grammar declares {one.name} and ast has no such class" + assert isinstance(node, type) + assert issubclass(node, ast.AST) + + +def test_every_constructor_in_the_grammar_is_a_class_in_ast(asdl): + """Section 2's first column, and that each one inherits from the type it belongs to.""" + for one in asdl.definitions: + for two in one.constructors: + node = getattr(ast, two.name, None) + assert node is not None, f"the grammar declares {two.name} and ast has no such class" + assert issubclass(node, getattr(ast, one.name)) + + +def test_the_field_order_is_the_order_the_grammar_declares(asdl): + """`_fields` is positional, so a reordering here breaks every hand built tree.""" + for one in asdl.definitions: + if not one.sum: + names = tuple(field.name for field in one.fields) + assert getattr(ast, one.name)._fields == names + continue + assert getattr(ast, one.name)._fields == () + for two in one.constructors: + names = tuple(field.name for field in two.fields) + assert getattr(ast, two.name)._fields == names + + +def test_the_attributes_are_the_ones_the_grammar_declares(asdl): + """Attributes are inherited from the type, so every constructor of a sum shares them.""" + for one in asdl.definitions: + names = tuple(field.name for field in one.attributes) + assert getattr(ast, one.name)._attributes == names + for two in one.constructors: + assert getattr(ast, two.name)._attributes == names + + +def test_the_defaults_are_the_ones_section_5_describes(asdl): + """Leaving a field out: `None`, an empty list, `Load()`, or `TypeError`.""" + for one in asdl.definitions: + for two in one.constructors: + for field in two.fields: + if field.kind == "required" and field.type != "expr_context": + continue + node = getattr(ast, two.name)(**_minimum(two)) + got = getattr(node, field.name) + if field.type == "expr_context": + assert isinstance(got, ast.Load) + elif field.sequence: + assert got == [] + else: + assert got is None + + +def test_an_attribute_is_never_required_and_optional_ones_still_default(asdl): + """Attributes come off the type, so this runs once per type that has any.""" + for one in asdl.definitions: + if not one.attributes: + continue + name = one.constructors[0].name if one.sum else one.name + node = getattr(ast, name)(**_minimum(one.constructors[0] if one.sum else one)) + for field in one.attributes: + if field.optional: + assert getattr(node, field.name) is None + else: + assert not hasattr(node, field.name) + + +def test_a_missing_line_number_is_refused_by_compile_rather_than_by_the_constructor(): + """Where INV-AST-008 is actually enforced, which is not where a reader expects.""" + built = ast.Module(body=[ast.Pass()], type_ignores=[]) + with pytest.raises(TypeError, match="lineno"): + compile(built, "", "exec") + positioned = ast.Module(body=[ast.Pass(lineno=1, col_offset=0)], type_ignores=[]) + assert compile(positioned, "", "exec").co_firstlineno == 1 + + +def test_a_required_field_left_out_raises_and_names_itself(): + """The other half of the rule above, which needs a node rather than a loop.""" + with pytest.raises(TypeError, match="name"): + ast.FunctionDef() + with pytest.raises(TypeError, match="args"): + ast.FunctionDef(name="f") + + +def test_a_sequence_default_is_not_shared_between_nodes(): + """An empty list per node, not one empty list handed out over and over.""" + first, second = ast.Module(), ast.Module() + first.body.append(ast.Pass()) + assert second.body == [] + + +def test_the_only_sequences_of_optional_are_the_two_section_6_names(asdl): + """`expr?* keys` and `expr?* kw_defaults`, the reason `Field.marks` exists at all.""" + found = [ + (one.name, field.name) + for one in asdl.definitions + for field in one.fields + if field.elements_optional + ] + found += [ + (two.name, field.name) + for one in asdl.definitions + for two in one.constructors + for field in two.fields + if field.elements_optional + ] + assert sorted(found) == [("Dict", "keys"), ("arguments", "kw_defaults")] + + +def test_a_none_key_in_a_dict_display_is_how_double_star_unpacking_is_written(): + """The first of the two, which has no node of its own and uses the gap instead.""" + node = ast.parse("{1: 2, **d}").body[0].value + assert node.keys[0].value == 1 + assert node.keys[1] is None + assert [type(one).__name__ for one in node.values] == ["Constant", "Name"] + + +def test_kw_defaults_lines_up_with_kwonlyargs_and_holds_none_for_the_gaps(): + """The second of the two, where the gap means an argument with no default.""" + args = ast.parse("def f(*, a, b=1): pass").body[0].args + assert [one.arg for one in args.kwonlyargs] == ["a", "b"] + assert args.kw_defaults[0] is None + assert isinstance(args.kw_defaults[1], ast.Constant) + + +def test_defaults_is_right_aligned_against_args_and_has_no_gaps(): + """The contrast that makes `kw_defaults` worth a section of its own.""" + args = ast.parse("def f(a, b=1): pass").body[0].args + assert [one.arg for one in args.args] == ["a", "b"] + assert len(args.defaults) == 1 + + +def test_every_field_type_is_a_definition_in_the_grammar_or_one_of_the_four(asdl): + """What lets section 2 print a type name without saying where to look it up.""" + names = {one.name for one in asdl.definitions} | set(BUILTIN_TYPES) + for one in asdl.definitions: + fields = list(one.fields) + list(one.attributes) + fields += [field for two in one.constructors for field in two.fields] + for field in fields: + assert field.type in names, f"{one.name}.{field.name} has type {field.type}" + + +def _minimum(constructor) -> dict[str, object]: + """The smallest set of arguments that builds this node, so the rest can be inspected.""" + return { + field.name: _value(field.type) + for field in constructor.fields + if field.kind == "required" and field.type != "expr_context" + } + + +def _value(name: str) -> object: + if name == "identifier": + return "x" + if name == "string": + return "x" + if name == "int": + return 0 + if name == "constant": + return None + return getattr(ast, name) diff --git a/tools/bpc/tests/test_bpc_model.py b/tools/bpc/tests/test_bpc_model.py new file mode 100644 index 0000000..1930708 --- /dev/null +++ b/tools/bpc/tests/test_bpc_model.py @@ -0,0 +1,167 @@ +"""The grammar as plain data, and the line numbers hung off it. + +The line numbers are the part worth testing hardest. Everything else in `bpc` is a +rearrangement of what `asdl.py` already worked out, but the walk that attaches lines is +`bpc`'s own guess about a file it did not parse, and a citation pointing at the wrong line +is the one failure this tool could produce that nobody would notice. +""" + +from __future__ import annotations + +import pytest + +from bpc.model import BUILTIN_TYPES, Field, Grammar, GrammarError, load_asdl, parse + + +def test_the_module_is_the_one_the_grammar_names(asdl): + assert asdl.name == "Python" + assert asdl.path == "Parser/Python.asdl" + + +def test_the_definitions_are_in_the_order_the_file_writes_them(asdl): + lines = [one.line for one in asdl.definitions] + assert lines == sorted(lines) + assert asdl.definitions[0].name == "mod" + + +def test_every_definition_is_reachable_by_name(asdl): + for one in asdl.definitions: + assert asdl.definition(one.name) is one + with pytest.raises(KeyError): + asdl.definition("NotAType") + + +def test_the_line_a_definition_is_cited_at_has_that_name_and_an_equals(asdl, tree): + """The self checking property the whole citation scheme rests on.""" + text = (tree / "Parser" / "Python.asdl").read_text(encoding="utf-8").splitlines() + for one in asdl.definitions: + line = text[one.line - 1] + assert one.name in line, f"{one.name} is not on line {one.line}: {line!r}" + assert "=" in line + + +def test_a_type_used_as_a_field_type_does_not_capture_the_definition(asdl): + """`arg` is a field type of `arguments` before it is a definition of its own. + + A forward scan for the first `arg` token lands on line 116, inside `arguments`, and + every citation for the type would then point at another type's field list. Requiring + an `=` after the name is what keeps them apart, and this is the test that says so. + """ + assert asdl.definition("arguments").line < asdl.definition("arg").line + assert asdl.definition("arg").line == 119 + + +def test_the_line_a_constructor_is_cited_at_has_that_name(asdl, tree): + text = (tree / "Parser" / "Python.asdl").read_text(encoding="utf-8").splitlines() + for one in asdl.definitions: + for two in one.constructors: + assert two.name in text[two.line - 1] + + +def test_a_constructor_is_inside_the_definition_that_declares_it(asdl): + for one in asdl.definitions: + for two in one.constructors: + assert one.line <= two.line <= one.end_line + + +def test_a_sum_has_constructors_and_a_product_has_fields(asdl): + for one in asdl.definitions: + assert one.sum == bool(one.constructors) + assert one.kind == ("sum" if one.sum else "product") + if not one.sum: + assert one.fields + + +def test_the_node_count_is_one_per_constructor_and_one_per_product(asdl): + expected = sum(len(one.constructors) or 1 for one in asdl.definitions) + assert asdl.node_count == expected + + +def test_every_field_kind_is_one_of_the_four_words(asdl): + words = {"required", "optional", "sequence", "sequence of optional"} + for one in asdl.definitions: + fields = list(one.fields) + list(one.attributes) + fields += [field for two in one.constructors for field in two.fields] + for field in fields: + assert field.kind in words + + +def test_notation_writes_the_field_back_the_way_the_grammar_wrote_it(): + assert Field("expr", "returns", optional=True, sequence=False, marks="?").notation == ( + "expr? returns" + ) + assert Field("stmt", "body", optional=False, sequence=True, marks="*").notation == ( + "stmt* body" + ) + assert Field("expr", "keys", optional=False, sequence=True, marks="?*").notation == ( + "expr?* keys" + ) + assert Field("identifier", "name", optional=False, sequence=False).notation == ( + "identifier name" + ) + + +def test_a_field_with_both_quantifiers_is_a_sequence_first(): + """`seq` wins over `opt`, because the value is a list before it is anything else.""" + field = Field("expr", "keys", optional=False, sequence=True, marks="?*") + assert field.elements_optional + assert field.kind == "sequence of optional" + assert not Field("expr", "body", optional=False, sequence=True, marks="*").elements_optional + + +def test_the_four_builtin_types_are_the_ones_that_are_not_defined(asdl): + declared = {one.name for one in asdl.definitions} + assert not (BUILTIN_TYPES & declared) + for field in asdl.definition("arg").fields: + assert field.builtin == (field.type in BUILTIN_TYPES) + + +def test_a_constructor_signature_reads_like_the_grammar(asdl): + node = next(one for one in asdl.definition("stmt").constructors if one.name == "Return") + assert node.signature == "Return(expr? value)" + pass_node = next(one for one in asdl.definition("stmt").constructors if one.name == "Pass") + assert pass_node.signature == "Pass" + + +def test_the_attributes_belong_to_the_type_not_the_constructor(asdl): + stmt = asdl.definition("stmt") + assert [one.name for one in stmt.attributes] == [ + "lineno", + "col_offset", + "end_lineno", + "end_col_offset", + ] + assert not asdl.definition("boolop").attributes + + +def test_a_tree_without_the_grammar_says_which_file_is_missing(tmp_path): + with pytest.raises(GrammarError, match=r"asdl\.py"): + parse(tmp_path) + + +def test_a_tree_with_asdl_but_no_grammar_says_so(tmp_path, tree): + (tmp_path / "Parser").mkdir() + (tmp_path / "Parser" / "asdl.py").write_text( + (tree / "Parser" / "asdl.py").read_text(encoding="utf-8"), encoding="utf-8" + ) + with pytest.raises(GrammarError, match=r"Python\.asdl"): + parse(tmp_path) + + +def test_asdl_is_imported_under_a_name_that_cannot_collide(tree): + module = load_asdl(tree) + assert module.__name__ == "bpc._cpython_asdl" + assert load_asdl(tree) is module + + +def test_the_grammar_is_read_once_per_tree(tree): + from bpc.model import grammar + + assert grammar(tree) is grammar(tree) + + +def test_a_grammar_can_be_built_by_hand_for_the_renderer_tests(): + """Nothing in `Grammar` needs a checkout, which is what keeps the render tests fast.""" + one = Grammar(name="Toy", definitions=(), line=1) + assert one.node_count == 0 + assert one.path == "Parser/Python.asdl" diff --git a/tools/bpc/tests/test_bpc_render.py b/tools/bpc/tests/test_bpc_render.py new file mode 100644 index 0000000..82179ec --- /dev/null +++ b/tools/bpc/tests/test_bpc_render.py @@ -0,0 +1,223 @@ +"""Grammar in, markdown out. + +Most of these run against a toy grammar built by hand rather than against CPython's. A +renderer tested only on the real thing is tested on 113 node kinds at once, and when it +breaks the failure is a diff of an 800 line file. The toy grammar is small enough that a +failing assertion names the thing that went wrong. + +The ones that do use the real grammar are the ones about scale: that every node kind +appears, that the counts in the prose match the tables under them, and that nothing is +quietly dropped. +""" + +from __future__ import annotations + +import pytest + +from bpc.model import Constructor, Definition, Field, Grammar +from bpc.render import BLOCKS, citation, conformance, nodes, observable, overview, table +from refcheck.citation import find_all +from refcheck.tree import PINNED_TAG + + +def field(name, type="expr", *, marks=""): + return Field(type=type, name=name, optional="?" in marks, sequence="*" in marks, marks=marks) + + +@pytest.fixture +def toy(): + """Two types: one sum with two constructors and attributes, one product.""" + return Grammar( + name="Toy", + line=1, + definitions=( + Definition( + name="stmt", + constructors=( + Constructor("Pass", (), line=5), + Constructor( + "Return", + (field("value", marks="?"), field("extras", marks="*")), + line=6, + ), + ), + fields=(), + attributes=(field("lineno", "int"),), + line=4, + end_line=7, + ), + Definition( + name="arg", + constructors=(), + fields=(field("name", "identifier"), field("annotation", marks="?")), + attributes=(), + line=9, + end_line=10, + ), + ), + ) + + +def test_a_citation_names_the_file_the_line_and_the_symbol(toy): + assert citation(toy, 4, "stmt") == f"`Parser/Python.asdl:4@{PINNED_TAG}#stmt`" + + +def test_every_citation_the_renderer_emits_is_one_refcheck_can_read(asdl): + """The check that matters: refcheck's own parser has to accept all of them.""" + text = "\n".join(BLOCKS[name](asdl) for name in ("overview", "nodes")) + found = find_all(text) + constructors = sum(len(one.constructors) for one in asdl.definitions) + assert len(found) == 2 * len(asdl.definitions) + constructors + 1 + for one in found: + assert one.tag == PINNED_TAG + assert one.path == "Parser/Python.asdl" + assert one.symbol + + +def test_a_table_gets_the_separator_markdown_insists_on(): + lines = table(["A", "B"], [["1", "2"]]) + assert lines == ["| A | B |", "|---|---|", "| 1 | 2 |"] + + +def test_a_table_with_no_rows_is_still_a_table(): + assert table(["A"], []) == ["| A |", "|---|"] + + +def test_the_overview_counts_what_the_grammar_holds(toy): + text = overview(toy) + assert "declares 2 types, 1 of them a choice" in text + assert "1 of them a single fixed shape" in text + assert "3 concrete node kinds" in text + assert "with 4 fields" in text + assert "1 of the types carry source location" in text + + +def test_the_overview_has_a_row_per_type_and_nothing_else(toy): + rows = [one for one in overview(toy).splitlines() if one.startswith("| ")] + assert len(rows) == 3 + assert "`stmt`" in rows[1] and "sum" in rows[1] + assert "`arg`" in rows[2] and "product" in rows[2] + + +def test_the_overview_counts_match_the_real_grammar(asdl): + text = overview(asdl) + assert f"declares {len(asdl.definitions)} types" in text + assert f"{asdl.node_count} concrete node kinds" in text + rows = [one for one in text.splitlines() if one.startswith("| ") and "sum" in one] + assert len(rows) == sum(1 for one in asdl.definitions if one.sum) + + +def test_a_sum_gets_one_row_per_field_and_repeats_the_node_name(toy): + text = nodes(toy) + assert "### 2.1 `stmt`" in text + assert "A choice between 2 constructors" in text + assert text.count("| `Return` |") == 2 + assert "| `Pass` | | no fields |" in text + + +def test_a_constructor_is_cited_once_rather_than_once_per_field(toy): + body = nodes(toy) + assert body.count(citation(toy, 6, "Return")) == 1 + + +def test_a_product_says_there_is_nothing_to_switch_on(toy): + text = nodes(toy) + assert "### 2.2 `arg`" in text + assert "A single shape with 2 fields" in text + assert "nothing to switch on" in text + + +def test_attributes_are_shown_separately_from_fields(toy): + text = nodes(toy) + assert "also carries 1 attributes" in text + assert "| Attribute | Type | Holds |" in text + + +def test_every_node_kind_in_the_real_grammar_gets_a_row(asdl): + text = nodes(asdl) + for one in asdl.definitions: + assert f"`{one.name}`" in text + for two in one.constructors: + assert f"| `{two.name}` |" in text + + +def test_the_field_kinds_reach_the_table(toy): + text = nodes(toy) + assert "| `value` | `expr` | optional |" in text + assert "| `extras` | `expr` | sequence |" in text + + +def test_a_sequence_of_optional_is_labelled_as_one(asdl): + text = nodes(asdl) + assert "| `keys` | `expr` | sequence of optional |" in text + assert "| `kw_defaults` | `expr` | sequence of optional |" in text + + +def test_the_observable_section_prints_fields_the_way_python_prints_them(toy): + text = observable(toy) + assert "| `stmt` | abstract | `()` | `('lineno',)` |" in text + assert "| `Return` | `stmt` | `('value', 'extras')` | `('lineno',)` |" in text + assert "| `arg` | concrete | `('name', 'annotation')` | `()` |" in text + + +def test_the_observable_section_states_all_three_defaults(toy): + text = observable(toy) + assert "raises `TypeError`" in text + assert "left out is `None`" in text + assert "a new empty list" in text + assert "`Load` singleton" in text + + +def test_the_observable_section_separates_the_two_kinds_of_attribute(toy): + text = observable(toy) + assert "none of them is ever required" in text + assert "default to `None`" in text + assert "`AttributeError`" in text + assert "until `compile` sees the tree" in text + + +def test_the_conformance_section_names_a_test_for_every_claim(toy): + text = conformance(toy) + rows = [one for one in text.splitlines() if one.startswith("| ")] + assert len(rows) == 7 + assert "`just citations`" in text + assert "The first five run under `just test`" in text + + +def test_the_conformance_counts_come_from_the_grammar(toy): + text = conformance(toy) + assert "| 2 types |" in text + assert "| 3 node kinds |" in text + assert "| 4 fields |" in text + assert "| 1 types carry attributes |" in text + assert "| 7 citations |" in text + + +def test_every_test_the_conformance_section_names_exists(): + """If this fails, section 8 is pointing at a test nobody can run.""" + from pathlib import Path + + source = Path(__file__).with_name("test_bpc_conformance.py").read_text(encoding="utf-8") + for line in conformance(Grammar("Toy", (), 1)).splitlines(): + for cell in line.split("|"): + name = cell.strip().strip("`") + if name.startswith("test_"): + assert f"def {name}(" in source, f"section 8 names {name} and it does not exist" + + +def test_the_renderer_is_deterministic(asdl): + for name, block in BLOCKS.items(): + assert block(asdl) == block(asdl), name + + +def test_no_block_comes_out_empty_on_the_real_grammar(asdl): + for name, block in BLOCKS.items(): + assert block(asdl).strip(), name + + +def test_nothing_the_renderer_writes_has_the_punctuation_the_project_bans(asdl): + for name, block in BLOCKS.items(): + text = block(asdl) + assert "\u2014" not in text, name + assert "\u2013" not in text, name + assert "\n---\n" not in text, name diff --git a/tools/bpc/tests/test_bpc_template.py b/tools/bpc/tests/test_bpc_template.py new file mode 100644 index 0000000..704de73 --- /dev/null +++ b/tools/bpc/tests/test_bpc_template.py @@ -0,0 +1,122 @@ +"""Swapping directives for blocks, and the four ways a source document can be wrong. + +The errors matter more than the happy path here. A template engine that quietly does +nothing when it does not recognise a directive produces a document with a heading and no +body, which reads like a subsystem with nothing to say rather than like a mistake. +""" + +from __future__ import annotations + +import pytest + +from bpc.model import Grammar +from bpc.template import BEGIN, END, Source, TemplateError, expand, find + + +@pytest.fixture +def empty(): + """A grammar with nothing in it, so the blocks come out short and readable.""" + return Grammar(name="Toy", definitions=(), line=1) + + +def source(tmp_path, text, name="BP-TOY.md"): + root = tmp_path / "blueprints" / "sources" + root.mkdir(parents=True) + path = root / name + path.write_text(text, encoding="utf-8") + return Source(path) + + +def test_a_source_knows_its_name_and_where_its_output_goes(tmp_path): + one = source(tmp_path, "\n") + assert one.name == "BP-TOY" + assert one.output == tmp_path / "blueprints" / "BP-TOY.md" + + +def test_the_blocks_are_listed_in_the_order_the_document_asks_for_them(tmp_path): + one = source(tmp_path, "a\n\nb\n\n") + assert one.blocks() == ["nodes", "overview"] + + +def test_a_directive_has_to_be_alone_on_its_line(tmp_path): + one = source(tmp_path, "text more\n\n") + assert one.blocks() == ["overview"] + + +def test_the_prose_around_a_directive_is_kept_exactly(tmp_path, empty): + one = source(tmp_path, "# Title\n\nbefore\n\n\n\nafter\n") + out = expand(one, empty).splitlines() + assert out[:4] == ["# Title", "", "before", ""] + assert out[-2:] == ["", "after"] + + +def test_the_output_marks_where_the_generated_part_starts_and_stops(tmp_path, empty): + out = expand(source(tmp_path, "\n"), empty) + assert out.startswith(BEGIN.format(name="overview") + "\n") + assert out.rstrip().endswith(END.format(name="overview")) + + +def test_the_directive_itself_does_not_survive_into_the_output(tmp_path, empty): + out = expand(source(tmp_path, "\n"), empty) + assert "" not in out + + +def test_the_output_ends_with_exactly_one_newline(tmp_path, empty): + out = expand(source(tmp_path, "\n\n\n\n"), empty) + assert out.endswith("\n") + assert not out.endswith("\n\n") + + +def test_an_unknown_block_names_the_line_and_lists_the_real_ones(tmp_path, empty): + one = source(tmp_path, "a\n\n") + with pytest.raises(TemplateError) as caught: + expand(one, empty) + assert ":2:" in str(caught.value) + assert "'invariants'" in str(caught.value) + assert "conformance, nodes, observable, overview" in str(caught.value) + + +def test_the_same_block_twice_is_an_error_rather_than_two_copies(tmp_path, empty): + one = source(tmp_path, "\n\n") + with pytest.raises(TemplateError, match="asked for twice"): + expand(one, empty) + + +def test_a_document_with_no_directives_is_not_a_source_document(tmp_path, empty): + with pytest.raises(TemplateError, match="does not need to be a source document"): + expand(source(tmp_path, "# Title\n\nAll hand written.\n"), empty) + + +def test_a_block_that_comes_out_empty_stops_the_build(tmp_path, empty): + """The renderer following a grammar it no longer understands, caught here.""" + from bpc import template + + original = dict(template.BLOCKS) + template.BLOCKS["nodes"] = lambda grammar: " \n" + try: + with pytest.raises(TemplateError, match="came out empty"): + expand(source(tmp_path, "\n"), empty) + finally: + template.BLOCKS.clear() + template.BLOCKS.update(original) + + +def test_finding_sources_ignores_anything_not_named_like_a_blueprint(tmp_path): + root = tmp_path / "sources" + root.mkdir() + for name in ("BP-AST.md", "BP-PARSER.md", "README.md", "notes.md"): + (root / name).write_text("\n", encoding="utf-8") + assert [one.name for one in find(root)] == ["BP-AST", "BP-PARSER"] + + +def test_finding_sources_in_a_directory_that_is_not_there_finds_nothing(tmp_path): + assert find(tmp_path / "nowhere") == [] + + +def test_the_real_source_document_expands_to_what_is_committed(asdl): + """The check that `just blueprints` runs, stated once here so it is covered by tests.""" + from pathlib import Path + + root = Path(__file__).resolve().parents[3] + one = Source(root / "blueprints" / "sources" / "BP-AST.md") + assert one.output.read_text(encoding="utf-8") == expand(one, asdl) diff --git a/uv.lock b/uv.lock index f48e7e9..5f26c4a 100644 --- a/uv.lock +++ b/uv.lock @@ -4,6 +4,7 @@ requires-python = ">=3.14" [manifest] members = [ + "bpc", "bpcheck", "cpython-internals", "nbbuild", @@ -135,6 +136,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, ] +[[package]] +name = "bpc" +version = "0.1.0" +source = { editable = "tools/bpc" } +dependencies = [ + { name = "refcheck" }, +] + +[package.metadata] +requires-dist = [{ name = "refcheck", editable = "tools/refcheck" }] + [[package]] name = "bpcheck" version = "0.1.0" @@ -312,6 +324,7 @@ name = "cpython-internals" version = "0.0.0" source = { virtual = "." } dependencies = [ + { name = "bpc" }, { name = "bpcheck" }, { name = "nbbuild" }, { name = "nbcheck" }, @@ -339,6 +352,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "bpc", editable = "tools/bpc" }, { name = "bpcheck", editable = "tools/bpcheck" }, { name = "nbbuild", editable = "tools/nbbuild" }, { name = "nbcheck", editable = "tools/nbcheck" },