feat(api): a stable crate api, and call_pure running every pure builtin without the vm

Cargo.toml

@@ -30,6 +30,10 @@ std = []
 coverage = []
 # The wrapper that makes compiler.wasm, its exports, host imports, allocator and panic handler.
 runtime = []
+# `call_pure` and the host of plain values it runs the builtins on, left out of the shipped compiler.wasm.
+pure = []
+# The modules behind the stable API, for the engine's own tests and tools that read them.
+internals = []
 
 [dependencies]
 wasm-abi = { workspace = true }

bench/.snapshot

@@ -2686,6 +2686,7 @@
     "5c2bc10687c51e47": [158905, 0.003931],
     "5c45dab6255cdfa2": [42115, 0.001092],
     "5c484bd4415e87a0": [30093, 0.000394],
+    "5c550b1fd0ac29f4": [514947, 0.010326],
     "5c553e7280389d16": [142059, 0.003278],
     "5c5b98a9293a3093": [202652335, 0.063319],
     "5c6bba93ffa792e3": [74334, 0.001504],
@@ -3322,6 +3323,7 @@
     "719fa45dde307837": [487645, 0.011391],
     "71be5632996ef365": [278527, 0.003983],
     "71d4dd885f926195": [6346730, 0.044801],
+    "71db7ceb1f6b7aff": [1057567, 0.022209],
     "71e0951cfc128284": [61225, 0.002116],
     "71f1ddd6928eae89": [110808, 0.003214],
     "71f1e207d8e1ab23": [50254, 0.001346],
@@ -5021,6 +5023,7 @@
     "a91ec33184d188a0": [172939, 0.002736],
     "a92177c469ced6d4": [286298, 0.005194],
     "a927435723c24ff4": [248567, 0.003412],
+    "a92c52d45c21a7a7": [107588, 0.003145],
     "a95987f5c0837746": [132760, 0.004216],
     "a96152270e64d52e": [32115, 0.000807],
     "a962501fe4ae6b5f": [146054, 0.003088],
@@ -5712,6 +5715,7 @@
     "c0dfa38aeb9f308f": [120868, 0.003118],
     "c0e0e77501467125": [77142, 0.000987],
     "c0ed36f21cebbc25": [154514, 0.002519],
+    "c0ef596fcfae5153": [633459, 0.012285],
     "c0f86155cec5ada9": [29542, 0.000395],
     "c0fea17c36a3bf71": [5658774, 0.195423],
     "c106b469b7ce6544": [143367, 0.002017],
@@ -6789,7 +6793,7 @@
     "e6e646fd2a4a0c16": [109598435, 0.013537],
     "e6e8e751bdb50cbd": [196511, 0.003984],
     "e7100f1cde6213a8": [39474, 0.000971],
-    "e71f49444f48944b": [730041027, 0.000408],
+    "e71f49444f48944b": [660041070, 0.000408],
     "e72c517833d63d53": [265765, 0.005015],
     "e732d3870c2eede9": [84194, 0.000616],
     "e73c84fcabca5c19": [61378, 0.000781],

cli/src/actor/actor.rs

@@ -3,7 +3,7 @@ use std::rc::Rc;
 use std::sync::{Arc, Mutex};
 
 use compiler::modules::dir_of;
-use compiler::vm::Limits;
+use compiler::embed::Limits;
 
 use crate::host::{driver, now_ns, Host, Project, Sink, Status, Vm, TICK_NS};
 use crate::pack::{base64_decode, Bundle, BUNDLE_TAG};

cli/src/actor/config.rs

@@ -1,4 +1,4 @@
-use compiler::vm::Limits;
+use compiler::embed::Limits;
 
 // The whole pool, groups plus the runtime knobs the CLI resolved from actor.yml.
 pub struct ActorConfig {

cli/src/actor/scheduler.rs

@@ -7,7 +7,7 @@ use std::time::{Duration, Instant};
 
 use slab::Slab;
 
-use compiler::vm::Limits;
+use compiler::embed::Limits;
 
 use crate::host::{now_ns, Host, Instance, Project, Runtime, Vm};
 

cli/src/cmd/actor.rs

@@ -1,7 +1,7 @@
 use anyhow::{anyhow, Context, Result};
 use crate::actor::{Group, Message, Out, ActorConfig};
 use crate::host::RunLimits;
-use compiler::vm::Limits;
+use compiler::embed::Limits;
 use serde::Deserialize;
 use std::path::Path;
 

cli/src/host/mod.rs

@@ -12,7 +12,7 @@ pub use resolver::{fetch_cached, Project};
 pub use vm::{Completion, Instance, Status, Vm};
 
 use anyhow::{anyhow, Result};
-use compiler::vm::types::Limits;
+use compiler::value::Limits;
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::path::PathBuf;

cli/src/host/vm.rs

@@ -1,7 +1,7 @@
 use super::{read, resolver, rt, stage, unstage, wt, Exports, Host, MemoryCap, Project, Sink, State};
 use anyhow::{anyhow, bail, Result};
 use compiler::abi::WireValue;
-use compiler::vm::Limits;
+use compiler::embed::Limits;
 use std::cell::{RefCell, RefMut};
 use std::collections::{HashMap, VecDeque};
 use std::rc::Rc;

docs/03-platforms/04-rust.mdx

@@ -44,6 +44,41 @@ A wasm that links the crate as a library, such as a plugin that only parses, dro
 edge-python = { git = "https://github.com/dylan-sutton-chavez/edge-python", tag = "v0.9.6", default-features = false }
 ```
 
+### The crate API
+
+The crate exposes these modules and keeps the rest to itself.
+
+| Module | Holds |
+|---|---|
+| `code` | `compile`, which lexes, parses and folds a source into an `SSAChunk`, the `Parser` for a resolver or a path, and the bytecode it makes, `Instruction`, `OpCode` with `is_jump` and `ALL`, `Value`, `Diagnostic` and `unpack_call` for the counts of a call operand. `Program::new(&chunk)` gives `scope_of(body)`, `params_of(fi)` and `bindings_of(body)` without running anything |
+| `types` | `BUILTIN_TYPES`, `builtin_type`, `exc_parents`, the method table `ALL_METHODS` with `lookup_method`, `CLASS_DUNDERS` with `class_dunder` for the special names a class body may define, and `NativeFnId` with the arity of each builtin |
+| `embed` | `VM` to run a chunk, `Limits`, `SchedulerStatus`, the snapshot calls `save`, `restore`, `source_of`, `limits_of`, `inspect_globals` and `inspect_stack`, and `Resolver`, `Resolved` and `NativeBinding` for modules and natives |
+| `value` | `Val`, `HeapObj`, `HeapPool`, `VmErr`, `BigInt`, `as_i128`, the allocator `meter` and the `Host` trait |
+| `pure` | `call_pure` with the `pure` feature on, `Host` always |
+| `lexer` | `lex`, `Token` and `TokenType` |
+| `modules`, `abi`, `util::sha256` | Manifests, locks and bundles, the wire format of plugins, and SHA-256 |
+
+`bindings_of(body)` answers for the module, every class and every function body with a `BodyKind` and a `Binding` per name slot. A function's names are `Local`, `Cell` or `Global`, the same answer `scope_of` gives. A class body's own names are `Member`, which its methods and comprehensions don't see, so they read past the class to the function around it or the module. Its other names are a `Cell` of that function, a `Lent` plain local of the function that defines the class, read by value as the class is made, or `Global`. Slots no variable takes, such as attribute names, are `Local`.
+
+A class body that defines a dunder outside `CLASS_DUNDERS` fails to compile. The list holds names no builtin type has as a method, such as `__set__`, `__delete__`, `__set_name__` and `__slots__`, so `ALL_METHODS` is no list of them.
+
+The `internals` feature opens the modules behind these for the engine's own tests and tools. They change with every release.
+
+### Builtins without a VM
+
+`call_pure` runs a builtin on plain values and returns its result or the exception it raised, with the class and the message the VM gives. No lexer, parser or VM runs, and the same body the VM runs gives the answer. It is behind the `pure` feature, so the shipped `compiler.wasm` carries none of it.
+
+```rust
+use compiler::pure::{Plain, call_pure};
+
+let up = call_pure("str.upper", &[Plain::Str("abc".into())]);
+assert_eq!(up, Ok(Plain::Str("ABC".into())));
+```
+
+A method is named `type.name` with its receiver first, and a dunder of a builtin value such as `str.__getitem__` resolves among the operators its type answers. `Plain` holds what a call takes and gives, ints of any size, floats, bools, `None`, `str`, `bytes`, `bytearray`, lists, tuples, dicts, sets, ranges and slices. A method or builtin function the tables mark pure runs, and any other answers `NotImplementedError`. `MethodDesc::excluded` and `NativeFnId::excluded` give the reason, such as a method that changes its receiver or one that may run user code.
+
+The bodies of the builtins are generic over `Host`, the heap they read and the user code only a VM runs. The VM is one host and `PlainHost` the other, so each builtin has one body and the VM compiles to the same code it did before.
+
 ## Imports
 
 `compiler.wasm` declares five imports from `env`.

docs/05-internals/01-design.mdx

@@ -106,6 +106,10 @@ A job keeps between calls whatever the builtin needs, so new state goes in a var
 
 Debug and coverage builds count the loops running in `VM::loops`. `exec_in` is the only way into `exec_frame`, and a loop opening inside another panics. The test `old_path_ceilings` runs every case of `vm.json` and fails when a generic frame is pushed. Release builds have no counter.
 
+## One body per builtin
+
+The methods and builtins read their values through `value::Host`, a trait over the heap and the few things only a VM has, its stack for results, its op budget, and the user iterators, dunders and exception state it keeps. `impl Host for VM` answers with the VM's own fields and inlines, so the VM compiles each body to the code it had before. `PlainHost` holds a heap with the kinds of Python and no user code, and `call_pure` runs a builtin on it. An entry of `ALL_METHODS` marked `pure` builds both forms of its body, and `MethodDesc::excluded` says why an entry has no plain form. The test `every_method_is_pure_or_says_why_not` keeps the table complete as builtins are added.
+
 ## Memory model
 
 `Val` is 64 bits and NaN-boxed, with `QNAN = 0x7FFC_0000_0000_0000` and `SIGN = 0x8000...`.

fuzz/src/main.rs

@@ -1,10 +1,10 @@
 use afl::fuzz;
 
 use compiler::lexer::lex;
-use compiler::parser::Parser;
-use compiler::vm::snapshot;
-use compiler::vm::types::{SchedulerStatus, VmErr};
-use compiler::vm::{Limits, VM};
+use compiler::code::Parser;
+use compiler::embed as snapshot;
+use compiler::value::{SchedulerStatus, VmErr};
+use compiler::embed::{Limits, VM};
 
 // Every run meters what it allocates, as compiler.wasm does.
 #[global_allocator]

lang/src/lib.rs

@@ -7,11 +7,11 @@ use core::cell::RefCell;
 
 use compiler::lexer::{lex, TokenType};
 use compiler::modules::{NativeBinding, Resolved, Resolver};
-use compiler::parser::{Diagnostic, Parser, SSAChunk};
+use compiler::code::{Diagnostic, Parser, SSAChunk};
 use compiler::value::{HeapObj, HeapPool, Val, VmErr, as_i128};
-use compiler::vm::VM;
+use compiler::embed::VM;
 
-pub use compiler::vm::Limits;
+pub use compiler::embed::Limits;
 
 // a source rewrite applied before lexing, its output must be valid edge python
 pub type Transform = dyn Fn(&str) -> String + Send + Sync + 'static;
@@ -242,7 +242,7 @@ impl Engine {
             }
             return Err(Error::Compile(buf, errs.first().map(|e| line_col(&src, e.start))));
         }
-        compiler::optimizer::constant_fold(&mut chunk);
+        compiler::code::constant_fold(&mut chunk);
         Ok(Program { chunk, source: src, limits: self.limits })
     }
 }

src/bridge.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::missing_safety_doc)] // safety contract in docs/03-reference/06-abi.mdx, per-fn sections would duplicate it
 
 use crate::abi::{classify_decode, classify_encode, DecodeBits, EncodeRequest, ErrorKind, ErrorStash, HandleTable, Op, PrimitiveBytes, TAG_INVALID};
-use crate::vm::types::{DictMap, HeapObj, Val, VmErr};
+use crate::vm::types::{DictMap, HeapObj, Host, Val, VmErr};
 use crate::vm::methods::lookup_method;
 use crate::vm::VM;
 use alloc::{rc::Rc, string::{String, ToString}, vec::Vec};
@@ -202,7 +202,7 @@ fn host_op(op: u32, recv: u32, name: &str, args: &[Val]) -> Result<Val, VmErr> {
         Some(Op::NewDict) => in_vm("edge_op new_dict called outside run()", |vm| vm.heap.alloc(HeapObj::Dict(Rc::new(RefCell::new(DictMap::new()))))),
         Some(Op::NewList) => in_vm("edge_op new_list called outside run()", |vm| vm.heap.alloc(HeapObj::List(Rc::new(RefCell::new(Vec::new()))))),
         Some(Op::TypeOf) => dispatch_type_of(recv),
-        Some(Op::NewTuple) => in_vm("edge_op new_tuple called outside run()", |vm| vm.tuple_from_items(args.to_vec())),
+        Some(Op::NewTuple) => in_vm("edge_op new_tuple called outside run()", |vm| vm.heap.alloc(HeapObj::Tuple(args.to_vec()))),
         Some(Op::NewSet) => in_vm("edge_op new_set called outside run()", |vm| vm.set_from_items(args.to_vec())),
         Some(Op::NewFrozenSet) => in_vm("edge_op new_frozenset called outside run()", |vm| vm.frozenset_from_items(args.to_vec())),
         // A system call names its plugin call in `recv`, the id `edge_call_id` gave it.

src/frontend/int.rs

@@ -401,6 +401,8 @@ pub fn range_slice(heap: &mut HeapPool, v: Val, start: Val, stop: Val, step: Val
 /* A slice index with a boxed int clamped to the inline extreme of its sign, which no sequence length reaches. */
 pub fn clip_index(heap: &HeapPool, v: Val) -> Val {
     let v = crate::value::unboxed(v, heap);
+    // A bool bounds as the int it is.
+    if v.is_bool() { return Val::int(v.as_bool() as i64); }
     if !is_big(heap, v) { return v; }
     Val::int(if view(heap, v).is_some_and(|b| b.is_neg()) { Val::INT_MIN } else { Val::INT_MAX })
 }

src/frontend/mod.rs

@@ -3,3 +3,10 @@ pub mod bigint;
 pub mod complex;
 pub mod sentinel;
 pub mod int;
+
+// Wide ints register first so no constant boxes one before them.
+pub fn install(heap: &mut crate::value::HeapPool) {
+    heap.set_wide_int(&int::KIND);
+    heap.know_kind(&complex::KIND);
+    heap.know_kind(&sentinel::KIND);
+}

src/lib.rs

@@ -13,12 +13,12 @@ pub mod wasm;
 
 /* Internal compiler helpers (not Edge Python stdlib), separate from pipeline code. */
 pub mod util {
-    pub mod hash;
-    pub mod fstr;
-    pub mod jesc;
+    pub(crate) mod hash;
+    pub(crate) mod fstr;
+    pub(crate) mod jesc;
     pub mod sha256;
-    pub mod uni;
-    pub mod suggest;
+    pub(crate) mod uni;
+    pub(crate) mod suggest;
 
     /* `Box::leak` for memory kept until exit, which Miri then holds as a static root instead of reporting it. */
     pub fn leak<T: ?Sized>(boxed: alloc::boxed::Box<T>) -> &'static mut T {
@@ -38,11 +38,78 @@ pub mod util {
 /* NaN-boxed values and the mark-and-sweep heap, the layer both the frontend and the VM build on. */
 pub mod value;
 
+// Public only through the internals feature for the engine's own tests and tools.
+macro_rules! internal {
+    ($($(#[$a:meta])* $m:ident;)*) => { $(
+        $(#[$a])* #[cfg(feature = "internals")] pub mod $m;
+        $(#[$a])* #[cfg(not(feature = "internals"))] pub(crate) mod $m;
+    )* };
+}
+
 pub mod lexer;
-pub mod parser;
-/* Post-SSA passes, run between parse and boot, touches no VM state. */
-pub mod optimizer;
-pub mod vm;
-/* The kinds Edge Python registers on the engine, the wide ints first. */
-pub mod frontend;
+internal! {
+    parser;
+    /* Post-SSA passes, run between parse and boot, touches no VM state. */
+    optimizer;
+    vm;
+    /* The kinds Edge Python registers on the engine, the wide ints first. */
+    frontend;
+}
+pub mod pure;
 pub mod modules;
+
+// Source to bytecode and the facts of compiled code read without running it.
+pub mod code {
+    pub use crate::parser::{
+        Parser, SSAChunk, Instruction, OpCode, Value, ImportKind, ImportEntry, Diagnostic, Lines, SsaName, ssa_strip, param_base_name, unpack_call,
+        COMP_ARG, SPREAD_BASES, INPLACE, ITER_VALUE, ITER_AS_IS, ITER_ASYNC, YIELD_FROM, SPREAD_ARGS, KEYWORDS,
+    };
+    pub use crate::optimizer::constant_fold;
+    pub use crate::vm::{Binding, BodyKind, FnScope, Kind, ParamKind};
+
+    // Lexes, parses and folds `src` into a chunk and its errors.
+    pub fn compile(src: &str) -> (SSAChunk, alloc::vec::Vec<Diagnostic>) {
+        let (tokens, _) = crate::lexer::lex(src);
+        let (mut chunk, errors) = Parser::new(src, tokens.into_iter()).parse();
+        constant_fold(&mut chunk);
+        (chunk, errors)
+    }
+
+    // Each body's scope and parameters resolved as the VM resolves them without running.
+    pub struct Program<'c> { vm: crate::vm::VM<'c>, bodies: crate::util::hash::FxHashMap<*const SSAChunk, (BodyKind, alloc::vec::Vec<Binding>)> }
+
+    impl<'c> Program<'c> {
+        pub fn new(chunk: &'c SSAChunk) -> Self {
+            let mut vm = crate::vm::VM::new(chunk);
+            let bodies = vm.body_bindings(chunk);
+            Self { vm, bodies }
+        }
+        // None for the module body.
+        pub fn scope_of(&self, body: &SSAChunk) -> Option<(usize, &FnScope)> { self.vm.scope_of(body) }
+        // Which body `body` is and the binding of each of its names, a module's, a class's or a function's, None for a chunk of another program.
+        pub fn bindings_of(&self, body: &SSAChunk) -> Option<(BodyKind, &[Binding])> { self.bodies.get(&(body as *const SSAChunk)).map(|(k, b)| (*k, b.as_slice())) }
+        // Kind and slot of each parameter in order.
+        pub fn params_of(&self, fi: usize) -> &[(ParamKind, usize)] { self.vm.params_of(fi) }
+    }
+}
+
+// Builtin types, methods and functions with arity and purity.
+pub mod types {
+    pub use crate::parser::{BUILTIN_TYPES, CLASS_DUNDERS, builtin_type, class_dunder};
+    pub use crate::vm::{ALL_METHODS, BuiltinMethodId, MethodDesc, MethodKind, lookup_method, exc_parents};
+    pub use crate::value::NativeFnId;
+}
+
+// A VM to embed, run and snapshot with host modules and natives.
+pub mod embed {
+    pub use crate::vm::VM;
+    pub use crate::vm::snapshot::{SnapErr, save, restore, source_of, limits_of, inspect_globals, inspect_stack};
+    pub use crate::value::{Limits, SchedulerStatus};
+    pub use crate::modules::{NativeBinding, Resolved, Resolver};
+}
+
+// Unstable internals for the engine's own tests and tools.
+#[cfg(feature = "internals")]
+pub mod internals {
+    pub use crate::{frontend, optimizer, parser, vm};
+}

src/parser/mod.rs

@@ -39,6 +39,9 @@ pub(super) const fn pack_call(pos: u16, kw: u16) -> u16 {
     ((kw & 0xFF) << 8) | (pos & 0xFF)
 }
 
+// Positional and keyword counts of a call operand without its flags.
+pub const fn unpack_call(operand: u16) -> (u16, u16) { (operand & 0xFF, (operand >> 8) & 0x3F) }
+
 // Operand of a binary op written as `op=`, so the VM asks `__iop__` first.
 pub const INPLACE: u16 = 1;