diff --git a/.claude/board/entries/2026-10-28-lane-layout-from-classview.md b/.claude/board/entries/2026-10-28-lane-layout-from-classview.md new file mode 100644 index 000000000..11887f94b --- /dev/null +++ b/.claude/board/entries/2026-10-28-lane-layout-from-classview.md @@ -0,0 +1,19 @@ +# Lane layout per class: chosen at compile time, resolved once through the ClassView + +**Status:** TEST-PINNED. Code: `crates/lance-graph-contract/src/lane_codec.rs`, `ClassView::epi_lanes`, `crates/cognitive-shader-driver/examples/thought_layers_probe.rs` (`bind_lanes::`, `write_epi`). Closes the open point of `2026-10-28-thought-layers-nan-skip.md` (the fixed lab layout). + +## Decision + +- The consumer fixes a class's lane layout at compile time as `EpiLaneClass::SPEC`, built with `checked()`: an invalid layout fails to compile. Fields: width (8 or 9 bits), code space (Sym / Asym), value offset, presence offset, lane count. +- The presence plane is mandatory. Absent (NaN, `$null`, UNKNOWN) is always presence 0 and value 0, at 8 and 9 bits alike. At 9 bits the presence bit is also the sign, so presence 0 with a nonzero magnitude is a negative value. A 9-bit lane must be Sym. +- ONNX `QuantizeLinear` / TFLite annotations pick the code space through `CodeSpace::from_zero_point` (0 = Sym). +- `ClassView::epi_lanes(class)` names the same constant for a classid. Default `None` = undeclared, and binding refuses, like `epistemic_reading`. +- A class is bound once (`bind_lanes::` returns a `LaneBinding`); writes take the binding and do not look the class up again. A binding shared across threads belongs in a `OnceLock` set when the registry loads; `LazyLock` does not fit, because its initializer cannot take the view. + +## Tests + +Contract: every validation rule refuses, adjacent regions pass, valid layouts pass, absent is exactly one of the 512 (value, presence) pairs, presence bits beyond the first byte, zero point to code space, a `compile_fail` doctest. Probe: binding refuses an undeclared or different layout; absent not written (DR512 and ONNX uint8); own value kept; 9-bit negative written; computed −0.499 written as +0; absent beside a change; asym code 0 written; sym int8 code −128 with presence 1 is a value; the contract rule agrees with `codec9` on all 512 codes. Eight disables, each red: bind check, keep step, presence-bit copy, presence-only test, value-only test, presence byte offset, overlap check, 9-bit Asym. + +## Open + +The classid → `ClassView::ClassId` (u16) mapping is the caller's; the probe takes `ClassId` as an argument rather than deriving it from the row key. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 454a77a4e..69350c4cb 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -495 entries, 2026-08-06 .. 2026-10-28. +496 entries, 2026-08-06 .. 2026-10-28. | date | entry id | finding | file | |---|---|---|---| | 2026-10-28 | `thought-layers-nan-skip` | | [2026-10-28-thought-layers-nan-skip.md](2026-10-28-thought-layers-nan-skip.md) | +| 2026-10-28 | `lane-layout-from-classview` | | [2026-10-28-lane-layout-from-classview.md](2026-10-28-lane-layout-from-classview.md) | | 2026-10-28 | `D-SAG-6-1` | | [2026-10-28-d-sag-6-1-audit-reference-ledger.md](2026-10-28-d-sag-6-1-audit-reference-ledger.md) | | 2026-10-28 | `D-DT64-6` | | [2026-10-28-d-dt64-6-routing-bindstore-behavior.md](2026-10-28-d-dt64-6-routing-bindstore-behavior.md) | | 2026-10-28 | `D-DT64-6-1a` | | [2026-10-28-d-dt64-6-ogit-predicates-impl.md](2026-10-28-d-dt64-6-ogit-predicates-impl.md) | diff --git a/crates/cognitive-shader-driver/examples/thought_layers_probe.rs b/crates/cognitive-shader-driver/examples/thought_layers_probe.rs index d42221286..f8f981d6f 100644 --- a/crates/cognitive-shader-driver/examples/thought_layers_probe.rs +++ b/crates/cognitive-shader-driver/examples/thought_layers_probe.rs @@ -31,7 +31,10 @@ use std::collections::HashMap; use lance_graph_contract::alpha::{AlphaAddr, AlphaAllocation, AlphaMask, AlphaOverlay}; use lance_graph_contract::canonical_node::{EdgeBlock, NodeGuid, NodeRow, TailVariant}; +use lance_graph_contract::class_view::{ClassId, ClassView}; +use lance_graph_contract::lane_codec::{EpiLaneClass, EpiLaneSpec, LaneWidth}; use ndarray::hpc::codec9::EpiLane9; +use std::marker::PhantomData; /// One thought's writes for [`ThoughtTables::write_parallel`]. pub type ThoughtJob = (ThoughtId, Vec<(AlphaAddr, [u8; 512])>); @@ -89,6 +92,9 @@ pub enum Refusal { KeyChanged, /// A slot-placed thought changed bytes outside its slot. OutsideSlot, + /// The class declares no lane layout, or a different one than the write + /// was compiled for. + LaneLayout, } /// One row's change against the base: the slots that differ and their XOR. @@ -185,84 +191,51 @@ pub struct Table { deltas: HashMap, } -/// Epistemic lanes per layout: one presence bit and one byte each. -pub const EPI_LANES: usize = 16; - -/// The width of a row's epistemic lanes. -/// -/// Both widths read "never touched" (NaN, `$null`, UNKNOWN) as presence 0 -/// and value 0. At U9 the presence bit is also the sign, so presence 0 with -/// a nonzero magnitude is a negative number (DR512, `ndarray::hpc::codec9`). -/// At U8 presence 0 with a nonzero value cannot be produced: it would mean -/// U9 bytes read through a U8 layout, which the layout type rules out at -/// compile time, so it is not checked per lane. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum LaneWidth { - U8, - U9, -} - -/// Where a row's epistemic lanes sit: [`EPI_LANES`] value bytes from -/// `values` and their presence bits, little-endian, in the two bytes at -/// `presence`. Lab layout: production reads it from the class's ClassView. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct EpiLanes { - width: LaneWidth, - values: usize, - presence: usize, +/// Whether a lane of `row` is absent (presence 0, value 0) under `spec`. A +/// 9-bit lane is read through `ndarray`'s DR512 codeword, so the probe and the +/// codec share one definition. +fn never_touched(spec: &EpiLaneSpec, row: &[u8; 512], lane: usize) -> bool { + let (p, m) = (spec.present(row, lane), spec.value(row, lane)); + match spec.width { + LaneWidth::Bits9 => EpiLane9::from_codeword(u16::from(m) | (u16::from(p) << 8)) + .is_some_and(EpiLane9::never_touched), + LaneWidth::Bits8 => EpiLaneSpec::absent(m, p), + } } -impl EpiLanes { - /// A layout outside the key slot whose value bytes and presence word do - /// not overlap; `None` otherwise. - #[must_use] - pub const fn new(width: LaneWidth, values: usize, presence: usize) -> Option { - let disjoint = values + EPI_LANES <= presence || presence + 2 <= values; - if values < SLOT - || presence < SLOT - || values + EPI_LANES > 512 - || presence + 2 > 512 - || !disjoint - { - return None; +/// Every absent lane of `new` takes the lane of `prior`: its value byte and +/// its presence bit. Returns the lanes kept. +fn keep_unknown(spec: &EpiLaneSpec, prior: &[u8; 512], new: &mut [u8; 512]) -> u64 { + let mut kept = 0u64; + for lane in 0..usize::from(spec.lanes) { + if never_touched(spec, new, lane) { + new[usize::from(spec.values) + lane] = prior[usize::from(spec.values) + lane]; + kept |= 1 << lane; } - Some(Self { - width, - values, - presence, - }) } - - fn present(&self, row: &[u8; 512], lane: usize) -> bool { - let word = u16::from_le_bytes([row[self.presence], row[self.presence + 1]]); - (word >> lane) & 1 == 1 + for lane in (0..usize::from(spec.lanes)).filter(|l| kept >> l & 1 == 1) { + let (byte, bit) = (usize::from(spec.presence) + lane / 8, 1u8 << (lane % 8)); + new[byte] = (new[byte] & !bit) | (prior[byte] & bit); } + kept +} - /// Presence 0 and value 0: the lane holds no value. - fn never_touched(&self, row: &[u8; 512], lane: usize) -> bool { - let (p, m) = (self.present(row, lane), row[self.values + lane]); - match self.width { - LaneWidth::U9 => EpiLane9::from_codeword(u16::from(m) | (u16::from(p) << 8)) - .is_some_and(EpiLane9::never_touched), - LaneWidth::U8 => !p && m == 0, - } - } +/// A class bound to the lane layout `L`: [`ThoughtTables::bind_lanes`] checked +/// once that the ClassView declares `L::SPEC` for the class. Writes take the +/// binding and do not look the class up again. A binding shared across +/// threads belongs in a `OnceLock` set when the registry loads; `LazyLock` +/// does not fit, because its initializer cannot take the view. +#[derive(Debug)] +pub struct LaneBinding { + class: ClassId, + layout: PhantomData, +} - /// Every never-touched lane of `new` takes the lane of `prior`, value - /// and presence bit together. Returns the lanes it kept. - fn keep_unknown(&self, prior: &[u8; 512], new: &mut [u8; 512]) -> u16 { - let mut kept = 0u16; - for lane in 0..EPI_LANES { - if self.never_touched(new, lane) { - new[self.values + lane] = prior[self.values + lane]; - kept |= 1 << lane; - } - } - let word = u16::from_le_bytes([new[self.presence], new[self.presence + 1]]); - let from = u16::from_le_bytes([prior[self.presence], prior[self.presence + 1]]); - let merged = (word & !kept) | (from & kept); - new[self.presence..self.presence + 2].copy_from_slice(&merged.to_le_bytes()); - kept +impl LaneBinding { + /// The class bound. + #[must_use] + pub fn class(&self) -> ClassId { + self.class } } @@ -443,27 +416,44 @@ impl<'a> ThoughtTables<'a> { }) } - /// A sparse write of epistemic lanes. A lane the thought leaves never - /// touched (presence 0, value 0) is not written: it keeps what the - /// thought read there before, so UNKNOWN never overwrites a known value - /// and the merge on read returns the earlier value. Every other lane, - /// including a U9 negative (presence 0, magnitude > 0) and +0, is - /// written as given. Bytes outside the lanes are written as by - /// [`write`](Self::write). + /// Bind `class` to the lane layout `L`, once. + /// + /// # Errors + /// [`Refusal::LaneLayout`] when `view` declares no layout or another one + /// for `class`. + pub fn bind_lanes( + view: &impl ClassView, + class: ClassId, + ) -> Result, Refusal> { + if view.epi_lanes(class) != Some(L::SPEC) { + return Err(Refusal::LaneLayout); + } + Ok(LaneBinding { + class, + layout: PhantomData, + }) + } + + /// A sparse write of epistemic lanes laid out as `L` declares. An absent + /// lane (presence 0, value 0) is not written: it keeps what the thought + /// read there before, so UNKNOWN never overwrites a known value and the + /// merge on read returns the earlier value. Every other lane, including a + /// 9-bit negative (presence 0, magnitude > 0) and +0, is written as given. + /// Bytes outside the lanes are written as by [`write`](Self::write). /// /// # Errors /// As [`write`](Self::write). - pub fn write_epi( + pub fn write_epi( &mut self, + _binding: &LaneBinding, thought: ThoughtId, addr: AlphaAddr, rung: u8, row: &[u8; 512], - lanes: &EpiLanes, ) -> Result { let prior = self.read(thought, addr)?; let mut new = *row; - lanes.keep_unknown(&prior, &mut new); + keep_unknown(&L::SPEC, &prior, &mut new); self.write(thought, addr, rung, &new) } @@ -889,125 +879,208 @@ mod tests { } // ── Epistemic lanes: NaN is not written, merge on read keeps the prior ── + // + // The layout of each class is a compile-time constant (`EpiLaneClass`); + // the ClassView names the same constant for the classid. + + use lance_graph_contract::lane_codec::{checked, CodeSpace}; + use lance_graph_contract::ontology::{DisplayTemplate, FieldRef}; + + const VALUES: u16 = 2 * SLOT as u16; + const PRESENCE: u16 = 3 * SLOT as u16; + const CLASS: ClassId = 7; + + /// DR512: 9-bit sign-magnitude, presence bit = sign. + struct Dr512; + impl EpiLaneClass for Dr512 { + const SPEC: EpiLaneSpec = checked(EpiLaneSpec { + width: LaneWidth::Bits9, + space: CodeSpace::Sym, + values: VALUES, + presence: PRESENCE, + lanes: 16, + }); + } - const VALUES: usize = 2 * SLOT; - const PRESENCE: usize = 3 * SLOT; + /// ONNX `QuantizeLinear` uint8 with zero point 128, plus a presence plane. + struct OnnxU8; + impl EpiLaneClass for OnnxU8 { + const SPEC: EpiLaneSpec = checked(EpiLaneSpec { + width: LaneWidth::Bits8, + space: CodeSpace::from_zero_point(128), + values: VALUES, + presence: PRESENCE, + lanes: 16, + }); + } - fn lanes(width: LaneWidth) -> EpiLanes { - EpiLanes::new(width, VALUES, PRESENCE).expect("lab layout") + /// TFLite symmetric int8 weights (zero point 0). + struct TfliteSym; + impl EpiLaneClass for TfliteSym { + const SPEC: EpiLaneSpec = checked(EpiLaneSpec { + width: LaneWidth::Bits8, + space: CodeSpace::from_zero_point(0), + values: VALUES, + presence: PRESENCE, + lanes: 16, + }); } - /// Base rows whose lanes all hold +(5 + lane): presence 1, value set. + /// A ClassView that declares one layout (or none) for every class. + struct View(Option); + impl ClassView for View { + fn fields(&self, _class: ClassId) -> &[FieldRef] { + &[] + } + fn template(&self, _class: ClassId) -> DisplayTemplate { + DisplayTemplate::Card + } + fn dolce_category_id(&self, _class: ClassId) -> u8 { + 0 + } + fn epi_lanes(&self, _class: ClassId) -> Option { + self.0 + } + } + + fn view() -> View { + View(Some(L::SPEC)) + } + + fn bound() -> LaneBinding { + ThoughtTables::bind_lanes::(&view::(), CLASS).expect("declared") + } + + /// Base rows whose lanes all hold value 5 + lane with presence 1. fn epi_rows() -> Vec { + let (v, p) = (usize::from(VALUES) - 32, usize::from(PRESENCE) - 32); let mut rows = base(8); for r in &mut rows { - for lane in 0..EPI_LANES { - r.value[VALUES - 32 + lane] = 5 + lane as u8; + for lane in 0..16 { + r.value[v + lane] = 5 + lane as u8; } - r.value[PRESENCE - 32..PRESENCE - 32 + 2].copy_from_slice(&u16::MAX.to_le_bytes()); + r.value[p..p + 2].copy_from_slice(&u16::MAX.to_le_bytes()); } rows } fn set_lane(row: &mut [u8; 512], lane: usize, present: bool, value: u8) { - row[VALUES + lane] = value; - let mut w = u16::from_le_bytes([row[PRESENCE], row[PRESENCE + 1]]); + let (v, p) = (usize::from(VALUES), usize::from(PRESENCE)); + row[v + lane] = value; + let mut w = u16::from_le_bytes([row[p], row[p + 1]]); w = (w & !(1 << lane)) | (u16::from(present) << lane); - row[PRESENCE..PRESENCE + 2].copy_from_slice(&w.to_le_bytes()); + row[p..p + 2].copy_from_slice(&w.to_le_bytes()); } fn lane(row: &[u8; 512], lane: usize) -> (bool, u8) { - let w = u16::from_le_bytes([row[PRESENCE], row[PRESENCE + 1]]); - ((w >> lane) & 1 == 1, row[VALUES + lane]) + let (v, p) = (usize::from(VALUES), usize::from(PRESENCE)); + let w = u16::from_le_bytes([row[p], row[p + 1]]); + ((w >> lane) & 1 == 1, row[v + lane]) + } + + fn one_thought(rows: &[NodeRow]) -> (AlphaAllocation<'_>, ThoughtId) { + (AlphaAllocation::over(rows), ThoughtId(1)) + } + + fn placed<'a>(alloc: &'a AlphaAllocation<'a>) -> ThoughtTables<'a> { + let mut tt = ThoughtTables::over(alloc, 1); + let t = tt.open_table().unwrap(); + tt.place(ThoughtId(1), Placement::Table(t)).unwrap(); + tt } + /// The contract's never-touched rule and ndarray's DR512 codeword agree on + /// all 512 codes. #[test] - fn layouts_over_the_key_or_overlapping_are_refused() { - assert!( - EpiLanes::new(LaneWidth::U9, 0, PRESENCE).is_none(), - "key slot" - ); - assert!( - EpiLanes::new(LaneWidth::U9, VALUES, VALUES + 8).is_none(), - "overlap" - ); - assert!( - EpiLanes::new(LaneWidth::U9, 500, PRESENCE).is_none(), - "past the row" + fn contract_and_codec9_agree_on_never_touched() { + for code in 0u16..512 { + let e = EpiLane9::from_codeword(code).unwrap(); + assert_eq!( + EpiLaneSpec::absent(e.magnitude(), e.presence()), + e.never_touched(), + "code {code:#x}" + ); + } + } + + /// Binding refuses a class with no declared layout or a different one; + /// the declared one binds once. + #[test] + fn binding_refuses_an_undeclared_or_different_layout() { + for v in [View(None), view::()] { + assert!(matches!( + ThoughtTables::bind_lanes::(&v, CLASS), + Err(Refusal::LaneLayout) + )); + } + let b = ThoughtTables::bind_lanes::(&view::(), CLASS).unwrap(); + assert_eq!(b.class(), CLASS); + } + + fn nan_over_known() { + let rows = epi_rows(); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); + let mut row = row_bytes(&rows[2]); + set_lane(&mut row, 4, false, 0); + let w = tt + .write_epi(&bound::(), th, rows[2].key, 1, &row) + .unwrap(); + assert_eq!(w, Written::NoChange); + assert_eq!(tt.stored_bytes(), 0); + assert_eq!( + lane(&tt.read(th, rows[2].key).unwrap(), 4), + (true, 9), + "base kept" ); - assert!(EpiLanes::new(LaneWidth::U8, VALUES, PRESENCE).is_some()); + assert!(tt.looked(th).unwrap().contains(2), "the look is recorded"); } - /// NaN over a known base value: the read returns the base value and - /// nothing is stored. Both widths. + /// NaN over a known base value: nothing stored, the base value is read. #[test] fn nan_over_a_known_value_is_not_written() { - for width in [LaneWidth::U8, LaneWidth::U9] { - let rows = epi_rows(); - let alloc = AlphaAllocation::over(&rows); - let mut tt = ThoughtTables::over(&alloc, 1); - let t = tt.open_table().unwrap(); - tt.place(ThoughtId(1), Placement::Table(t)).unwrap(); - let mut row = row_bytes(&rows[2]); - set_lane(&mut row, 4, false, 0); - let w = tt - .write_epi(ThoughtId(1), rows[2].key, 1, &row, &lanes(width)) - .unwrap(); - assert_eq!(w, Written::NoChange, "{width:?}"); - assert_eq!(tt.stored_bytes(), 0, "{width:?}"); - let read = tt.read(ThoughtId(1), rows[2].key).unwrap(); - assert_eq!(lane(&read, 4), (true, 9), "{width:?}: base value kept"); - assert!( - tt.looked(ThoughtId(1)).unwrap().contains(2), - "the look is recorded" - ); - } + nan_over_known::(); + nan_over_known::(); } - /// NaN after the thought's own write keeps what the thought wrote, not - /// the base: the merge on read returns the earlier value. + /// NaN after the thought's own write keeps what the thought wrote. #[test] fn nan_after_an_own_write_keeps_the_own_value() { let rows = epi_rows(); - let alloc = AlphaAllocation::over(&rows); - let mut tt = ThoughtTables::over(&alloc, 1); - let t = tt.open_table().unwrap(); - tt.place(ThoughtId(1), Placement::Table(t)).unwrap(); - let l = lanes(LaneWidth::U9); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); + let b = bound::(); let mut row = row_bytes(&rows[1]); set_lane(&mut row, 3, true, 40); - tt.write_epi(ThoughtId(1), rows[1].key, 1, &row, &l) - .unwrap(); + tt.write_epi(&b, th, rows[1].key, 1, &row).unwrap(); set_lane(&mut row, 3, false, 0); - let w = tt - .write_epi(ThoughtId(1), rows[1].key, 1, &row, &l) - .unwrap(); + let w = tt.write_epi(&b, th, rows[1].key, 1, &row).unwrap(); assert_eq!(w, Written::NoChange); - let read = tt.read(ThoughtId(1), rows[1].key).unwrap(); - assert_eq!(lane(&read, 3), (true, 40), "own write kept, not the base 8"); + assert_eq!( + lane(&tt.read(th, rows[1].key).unwrap(), 3), + (true, 40), + "not the base 8" + ); } - /// U9: presence 0 with magnitude 5 is −5, written and read back. + /// 9-bit: presence 0 with magnitude 5 is -5, written and read back. #[test] - fn a_u9_negative_is_written() { + fn a_nine_bit_negative_is_written() { let rows = epi_rows(); - let alloc = AlphaAllocation::over(&rows); - let mut tt = ThoughtTables::over(&alloc, 1); - let t = tt.open_table().unwrap(); - tt.place(ThoughtId(1), Placement::Table(t)).unwrap(); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); let mut row = row_bytes(&rows[0]); set_lane(&mut row, 6, false, 5); let w = tt - .write_epi(ThoughtId(1), rows[0].key, 1, &row, &lanes(LaneWidth::U9)) + .write_epi(&bound::(), th, rows[0].key, 1, &row) .unwrap(); assert!(matches!(w, Written::Stored { .. })); - let (p, m) = lane(&tt.read(ThoughtId(1), rows[0].key).unwrap(), 6); - let v = EpiLane9::from_codeword(u16::from(m) | (u16::from(p) << 8)).unwrap(); - assert_eq!(v.value(), Some(-5)); + let (p, m) = lane(&tt.read(th, rows[0].key).unwrap(), 6); + let e = EpiLane9::from_codeword(u16::from(m) | (u16::from(p) << 8)).unwrap(); + assert_eq!(e.value(), Some(-5)); } - /// A computed −0.499 is encoded as +0 in the same step and is written: - /// +0 is a value, never NaN. + /// A computed -0.499 is encoded as +0 in the same step and is written. #[test] fn a_computed_negative_zero_is_written_as_plus_zero() { use ndarray::hpc::codec8::{Calibration, TiesEven}; @@ -1020,47 +1093,74 @@ mod tests { let code = Codec9::::encode_touched(-0.499).unwrap(); assert_eq!((code.presence(), code.magnitude()), (true, 0)); let rows = epi_rows(); - let alloc = AlphaAllocation::over(&rows); - let mut tt = ThoughtTables::over(&alloc, 1); - let t = tt.open_table().unwrap(); - tt.place(ThoughtId(1), Placement::Table(t)).unwrap(); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); let mut row = row_bytes(&rows[5]); set_lane(&mut row, 0, code.presence(), code.magnitude()); let w = tt - .write_epi(ThoughtId(1), rows[5].key, 1, &row, &lanes(LaneWidth::U9)) + .write_epi(&bound::(), th, rows[5].key, 1, &row) .unwrap(); assert!( matches!(w, Written::Stored { .. }), "+0 replaces the base 5" ); - let read = tt.read(ThoughtId(1), rows[5].key).unwrap(); - assert_eq!(lane(&read, 0), (true, 0)); + assert_eq!(lane(&tt.read(th, rows[5].key).unwrap(), 0), (true, 0)); } - /// A NaN lane next to a changed lane: only the changed lane is stored, - /// and the NaN lane reads its prior value. + fn nan_beside_change() { + let rows = epi_rows(); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); + let mut row = row_bytes(&rows[4]); + set_lane(&mut row, 1, false, 0); + set_lane(&mut row, 2, true, 99); + let w = tt + .write_epi(&bound::(), th, rows[4].key, 1, &row) + .unwrap(); + assert_eq!(w, Written::Stored { slots: 1 << 2 }, "value slot only"); + let read = tt.read(th, rows[4].key).unwrap(); + assert_eq!(lane(&read, 1), (true, 6)); + assert_eq!(lane(&read, 2), (true, 99)); + } + + /// A NaN lane next to a changed lane: only the change is stored. #[test] fn nan_beside_a_change_stores_only_the_change() { - for width in [LaneWidth::U8, LaneWidth::U9] { - let rows = epi_rows(); - let alloc = AlphaAllocation::over(&rows); - let mut tt = ThoughtTables::over(&alloc, 1); - let t = tt.open_table().unwrap(); - tt.place(ThoughtId(1), Placement::Table(t)).unwrap(); - let mut row = row_bytes(&rows[4]); - set_lane(&mut row, 1, false, 0); - set_lane(&mut row, 2, true, 99); - let w = tt - .write_epi(ThoughtId(1), rows[4].key, 1, &row, &lanes(width)) - .unwrap(); - assert_eq!( - w, - Written::Stored { slots: 1 << 2 }, - "{width:?}: value slot only" - ); - let read = tt.read(ThoughtId(1), rows[4].key).unwrap(); - assert_eq!(lane(&read, 1), (true, 6), "{width:?}"); - assert_eq!(lane(&read, 2), (true, 99), "{width:?}"); - } + nan_beside_change::(); + nan_beside_change::(); + } + + /// Asymmetric 8-bit with a presence plane: value 0 with presence 1 is a + /// code like any other and is written. + #[test] + fn an_asym_zero_code_is_written() { + let rows = epi_rows(); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); + let mut row = row_bytes(&rows[3]); + set_lane(&mut row, 5, true, 0); + let w = tt + .write_epi(&bound::(), th, rows[3].key, 1, &row) + .unwrap(); + assert!(matches!(w, Written::Stored { .. })); + assert_eq!(lane(&tt.read(th, rows[3].key).unwrap(), 5), (true, 0)); + } + + /// Symmetric int8: absent is presence 0 and value 0 like every layout; + /// code -128 with presence 1 is a value and is written. + #[test] + fn a_sym_int8_min_code_is_a_value() { + let rows = epi_rows(); + let (alloc, th) = one_thought(&rows); + let mut tt = placed(&alloc); + let b = bound::(); + let mut row = row_bytes(&rows[6]); + set_lane(&mut row, 7, false, 0); + let w = tt.write_epi(&b, th, rows[6].key, 1, &row).unwrap(); + assert_eq!(w, Written::NoChange, "absent: base kept"); + set_lane(&mut row, 7, true, 0x80); + let w = tt.write_epi(&b, th, rows[6].key, 1, &row).unwrap(); + assert!(matches!(w, Written::Stored { .. })); + assert_eq!(lane(&tt.read(th, rows[6].key).unwrap(), 7), (true, 0x80)); } } diff --git a/crates/lance-graph-contract/src/class_view.rs b/crates/lance-graph-contract/src/class_view.rs index 3674691e4..db5a4c65e 100644 --- a/crates/lance-graph-contract/src/class_view.rs +++ b/crates/lance-graph-contract/src/class_view.rs @@ -1358,6 +1358,20 @@ pub trait ClassView { None } + /// The epistemic lane layout this class declares + /// ([`lane_codec`](crate::lane_codec)): lane width, code space, and where + /// the value and presence bytes sit (absent = presence 0, value 0). The + /// consumer fixes it at compile time as an + /// [`EpiLaneClass::SPEC`](crate::lane_codec::EpiLaneClass::SPEC); this + /// lookup names that constant for a classid. `None` = undeclared, and a + /// writer REFUSES: a lane read under a guessed layout could take a + /// negative 9-bit value for absent. Selection only; never changes + /// stored bytes. + #[inline] + fn epi_lanes(&self, _class: ClassId) -> Option { + None + } + /// Which value-slab schema preset this class materialises in /// [`NodeRow::value`](crate::canonical_node::NodeRow::value). /// diff --git a/crates/lance-graph-contract/src/lane_codec.rs b/crates/lance-graph-contract/src/lane_codec.rs new file mode 100644 index 000000000..b6083fc79 --- /dev/null +++ b/crates/lance-graph-contract/src/lane_codec.rs @@ -0,0 +1,349 @@ +//! Epistemic lane layout of a class, chosen by the consumer at compile time. +//! +//! A class that carries quantized values in its row declares, once, how its +//! lanes are stored: their width, the code space, and where the value bytes +//! and the presence bits sit. The consumer picks this at compile time, +//! typically from the quantization annotations it already has (ONNX +//! `QuantizeLinear` / TFLite tensor quantization: a zero point of 0 is +//! [`CodeSpace::Sym`], any other zero point is [`CodeSpace::Asym`]), and +//! states it as a constant: +//! +//! ``` +//! use lance_graph_contract::lane_codec::{ +//! checked, CodeSpace, EpiLaneClass, EpiLaneSpec, LaneWidth, +//! }; +//! +//! /// A class whose lanes are ONNX uint8 values with zero point 128. +//! struct SensorReading; +//! impl EpiLaneClass for SensorReading { +//! const SPEC: EpiLaneSpec = checked(EpiLaneSpec { +//! width: LaneWidth::Bits8, +//! space: CodeSpace::from_zero_point(128), +//! values: 64, +//! presence: 96, +//! lanes: 16, +//! }); +//! } +//! assert_eq!(SensorReading::SPEC.space, CodeSpace::Asym); +//! ``` +//! +//! A combination that has no meaning does not compile: +//! +//! ```compile_fail,E0080 +//! use lance_graph_contract::lane_codec::{ +//! checked, CodeSpace, EpiLaneClass, EpiLaneSpec, LaneWidth, +//! }; +//! struct Bad; +//! impl EpiLaneClass for Bad { +//! // A 9-bit lane is sign-magnitude: it has no zero point. +//! const SPEC: EpiLaneSpec = checked(EpiLaneSpec { +//! width: LaneWidth::Bits9, +//! space: CodeSpace::Asym, +//! values: 64, +//! presence: 96, +//! lanes: 16, +//! }); +//! } +//! let _ = Bad::SPEC; +//! ``` +//! +//! [`ClassView::epi_lanes`](crate::class_view::ClassView::epi_lanes) names the +//! same constant for a class, so a reader holding only a classid can resolve +//! it, and a writer generic over [`EpiLaneClass`] can refuse a class that +//! declares a different layout. +//! +//! # Absent +//! +//! Every lane has a presence bit; the presence plane is not optional. A lane +//! is absent (never touched: NaN, `$null`, UNKNOWN) when its presence bit is 0 +//! **and** its value is 0, at 8 and 9 bits alike. At 9 bits the presence bit +//! is also the sign (DR512, `ndarray` `hpc::codec9`), so presence 0 with a +//! nonzero magnitude is a negative value. At 8 bits presence 0 with a nonzero +//! value cannot arise except by reading a 9-bit lane through an 8-bit layout; +//! the width in this spec is what rules that out. +//! +//! A sparse write leaves an absent lane unwritten, so it keeps the value read +//! before. This module describes layout only; it stores nothing and adds no +//! bytes to a row. + +/// Bits per lane. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum LaneWidth { + /// One value byte and one presence bit per lane. + Bits8, + /// One magnitude byte per lane plus its presence/sign bit (DR512). + Bits9, +} + +/// The code space of a lane. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CodeSpace { + /// Zero point 0: values are symmetric around code 0. + Sym, + /// A nonzero zero point. + Asym, +} + +impl CodeSpace { + /// The code space of an ONNX / TFLite quantization annotation. + #[must_use] + pub const fn from_zero_point(zero_point: i32) -> Self { + if zero_point == 0 { + Self::Sym + } else { + Self::Asym + } + } +} + +/// Where a class keeps its epistemic lanes, and how they are coded. +/// +/// Offsets are bytes into the 512-byte row (key, edge block, value slab). +/// Presence bits are little-endian, bit `i` for lane `i`, in +/// `lanes.div_ceil(8)` bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EpiLaneSpec { + pub width: LaneWidth, + pub space: CodeSpace, + /// First value byte; one byte per lane. + pub values: u16, + /// First presence byte. + pub presence: u16, + /// Number of lanes, `1..=64`. + pub lanes: u8, +} + +/// Bytes of a row. +pub const ROW_BYTES: u16 = 512; +/// First byte of the value slab: lanes never overlap the key or edge block. +pub const VALUE_SLAB_START: u16 = 32; + +/// Why an [`EpiLaneSpec`] is refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpecError { + /// `lanes` is 0 or above 64. + LaneCount, + /// A region starts before the value slab or ends past the row. + OutOfSlab, + /// The value bytes and the presence bytes overlap. + Overlap, + /// A 9-bit lane is sign-magnitude and therefore symmetric. + NineBitIsSym, +} + +impl EpiLaneSpec { + /// Bytes of the presence plane. + #[must_use] + pub const fn presence_bytes(&self) -> u16 { + (self.lanes as u16).div_ceil(8) + } + + /// Checks the spec against the rules above. + /// + /// # Errors + /// The first rule broken. + pub const fn validate(&self) -> Result<(), SpecError> { + if self.lanes == 0 || self.lanes > 64 { + return Err(SpecError::LaneCount); + } + let v_end = self.values + self.lanes as u16; + let p_end = self.presence + self.presence_bytes(); + if self.values < VALUE_SLAB_START + || v_end > ROW_BYTES + || self.presence < VALUE_SLAB_START + || p_end > ROW_BYTES + { + return Err(SpecError::OutOfSlab); + } + if self.presence < v_end && self.values < p_end { + return Err(SpecError::Overlap); + } + match (self.width, self.space) { + (LaneWidth::Bits9, CodeSpace::Asym) => Err(SpecError::NineBitIsSym), + _ => Ok(()), + } + } + + /// Whether a lane holding `value` with presence bit `present` is absent: + /// presence 0 and value 0. + #[must_use] + pub const fn absent(value: u8, present: bool) -> bool { + !present && value == 0 + } + + /// Lane `lane`'s value byte in `row`. + #[must_use] + pub fn value(&self, row: &[u8], lane: usize) -> u8 { + row[self.values as usize + lane] + } + + /// Lane `lane`'s presence bit in `row`. + #[must_use] + pub fn present(&self, row: &[u8], lane: usize) -> bool { + (row[self.presence as usize + lane / 8] >> (lane % 8)) & 1 == 1 + } +} + +/// Returns `spec`, or fails to compile when it is used in a constant and +/// breaks a rule of [`EpiLaneSpec::validate`]. +#[must_use] +pub const fn checked(spec: EpiLaneSpec) -> EpiLaneSpec { + match spec.validate() { + Ok(()) => spec, + Err(SpecError::LaneCount) => panic!("EpiLaneSpec: lanes must be 1..=64"), + Err(SpecError::OutOfSlab) => panic!("EpiLaneSpec: lanes must lie in the value slab"), + Err(SpecError::Overlap) => panic!("EpiLaneSpec: value and presence bytes overlap"), + Err(SpecError::NineBitIsSym) => panic!("EpiLaneSpec: a 9-bit lane is Sym"), + } +} + +/// A class's lane layout, fixed at compile time. +pub trait EpiLaneClass { + /// Build with [`checked`] so an invalid layout fails to compile. + const SPEC: EpiLaneSpec; +} + +#[cfg(test)] +mod tests { + use super::*; + + const DR512: EpiLaneSpec = EpiLaneSpec { + width: LaneWidth::Bits9, + space: CodeSpace::Sym, + values: 64, + presence: 96, + lanes: 16, + }; + + #[test] + fn valid_layouts_pass() { + assert_eq!(DR512.validate(), Ok(())); + for space in [CodeSpace::Sym, CodeSpace::Asym] { + let b8 = EpiLaneSpec { + width: LaneWidth::Bits8, + space, + ..DR512 + }; + assert_eq!(b8.validate(), Ok(())); + } + } + + #[test] + fn each_rule_refuses() { + let e = |s: EpiLaneSpec| s.validate().unwrap_err(); + assert_eq!(e(EpiLaneSpec { lanes: 0, ..DR512 }), SpecError::LaneCount); + assert_eq!(e(EpiLaneSpec { lanes: 65, ..DR512 }), SpecError::LaneCount); + assert_eq!( + e(EpiLaneSpec { + values: 16, + ..DR512 + }), + SpecError::OutOfSlab + ); + assert_eq!( + e(EpiLaneSpec { + values: 500, + ..DR512 + }), + SpecError::OutOfSlab + ); + assert_eq!( + e(EpiLaneSpec { + presence: 16, + ..DR512 + }), + SpecError::OutOfSlab + ); + assert_eq!( + e(EpiLaneSpec { + presence: 511, + ..DR512 + }), + SpecError::OutOfSlab + ); + assert_eq!( + e(EpiLaneSpec { + presence: 70, + ..DR512 + }), + SpecError::Overlap + ); + assert_eq!( + e(EpiLaneSpec { + presence: 63, + ..DR512 + }), + SpecError::Overlap + ); + assert_eq!( + e(EpiLaneSpec { + space: CodeSpace::Asym, + ..DR512 + }), + SpecError::NineBitIsSym + ); + } + + /// Adjacent regions do not overlap: presence ending where the values + /// start, and presence starting where the values end, both pass. + #[test] + fn adjacent_regions_pass() { + assert_eq!( + EpiLaneSpec { + presence: 62, + ..DR512 + } + .validate(), + Ok(()) + ); + assert_eq!( + EpiLaneSpec { + presence: 80, + ..DR512 + } + .validate(), + Ok(()) + ); + } + + #[test] + fn presence_bytes_round_up() { + assert_eq!(EpiLaneSpec { lanes: 1, ..DR512 }.presence_bytes(), 1); + assert_eq!(EpiLaneSpec { lanes: 9, ..DR512 }.presence_bytes(), 2); + assert_eq!(EpiLaneSpec { lanes: 64, ..DR512 }.presence_bytes(), 8); + } + + #[test] + fn zero_point_selects_the_code_space() { + assert_eq!(CodeSpace::from_zero_point(0), CodeSpace::Sym); + assert_eq!(CodeSpace::from_zero_point(128), CodeSpace::Asym); + assert_eq!(CodeSpace::from_zero_point(-3), CodeSpace::Asym); + } + + /// Presence 0 and value 0 is the only absent code; a 9-bit negative + /// (presence 0, value > 0) and +0 (presence 1, value 0) are values. + #[test] + fn absent_is_presence_zero_and_value_zero() { + assert!(EpiLaneSpec::absent(0, false)); + assert!(!EpiLaneSpec::absent(5, false), "9-bit -5"); + assert!(!EpiLaneSpec::absent(0, true), "+0"); + assert_eq!( + (0..=255u8) + .flat_map(|v| [(v, false), (v, true)]) + .filter(|&(v, p)| EpiLaneSpec::absent(v, p)) + .count(), + 1 + ); + } + + #[test] + fn presence_bits_are_read_little_endian_past_the_first_byte() { + let mut row = [0u8; 512]; + row[96] = 0b0000_0100; + row[97] = 0b0000_0010; + assert!(DR512.present(&row, 2)); + assert!(DR512.present(&row, 9)); + assert!(!DR512.present(&row, 1)); + assert!(!DR512.present(&row, 8)); + } +} diff --git a/crates/lance-graph-contract/src/lib.rs b/crates/lance-graph-contract/src/lib.rs index 50fb31184..ee9b459f8 100644 --- a/crates/lance-graph-contract/src/lib.rs +++ b/crates/lance-graph-contract/src/lib.rs @@ -119,6 +119,7 @@ pub mod hotplug; pub mod identity_quad; pub mod jit; pub mod kanban; +pub mod lane_codec; pub mod legacy_outliers; pub mod literal_graph; pub mod mail;