Gap stage 1: append-only blocks for inode values and binding hints, schema v2 (#91)

core/crates/layerfs-storage/README.md

@@ -120,7 +120,8 @@ destructor.
 
 - Filesystem trees, runtime adapters and cloud placement are later scope. Payload
   DELTA chains and pooled metadata are implemented.
-- Inode values are rows of `inode_value` (ordinal, 73 value bytes). The earlier
+- Inode values are stored per ordinal through `PackPersistence` (73 value bytes);
+  the engine keeps them in append-only blocks since its schema version 2 (#91). The earlier
   value-group catalogue and v6 value packs are no longer written or read; a
   Store that has them is refused at open.
 - The retained index window boundary (131,072 entries) and its cold-start replay

core/crates/layerfs-storage/src/save/pooled.rs

@@ -33,14 +33,20 @@ struct Base {
     chain_encoded: u64,
 }
 
-/// Candidate filter of one value: eight bytes of each of its two digests.
+/// Candidate filter of one value: a 64-bit mix of all 73 bytes. A filter of
+/// only some bytes let two values that differ elsewhere (the same content and
+/// mode with another mtime) share one memo entry, so the second one's later
+/// copies in the same Save each took a new ordinal (#91).
 fn fingerprint(value: &[u8; INODE_VALUE_BYTES]) -> u64 {
-    let word = |from: usize| {
-        let mut bytes = [0_u8; 8];
-        bytes.copy_from_slice(&value[from..from + 8]);
-        u64::from_le_bytes(bytes)
-    };
-    word(INODE_VALUE_BYTES - 64) ^ word(INODE_VALUE_BYTES - 32).rotate_left(29) ^ word(0)
+    value
+        .chunks(8)
+        .fold(0x9e37_79b9_7f4a_7c15_u64, |hash, chunk| {
+            let mut word = [0_u8; 8];
+            word[..chunk.len()].copy_from_slice(chunk);
+            (hash ^ u64::from_le_bytes(word))
+                .wrapping_mul(0x0000_0100_0000_01b3)
+                .rotate_left(29)
+        })
 }
 
 impl State<'_> {

core/crates/layerfs-storage/tests/stored_value_reads.rs

@@ -257,3 +257,68 @@ fn one_handle_reads_any_number_of_leaves_at_one_lookup_each() {
     }
     assert_eq!(value_lookups(&metadata) as u64, LEAVES);
 }
+
+/// A leaf of `ROWS` files of one content and mode whose mtime differs from
+/// `mtime` only in its low bytes and nanoseconds by `bump`.
+fn inline_leaf(first_serial: u64, bump: u32) -> FinalizedObject {
+    use layerfs_content::{
+        filesystem::attributes::portable::PortableMetadata, inode_leaf::InodeMetadata,
+    };
+    let rows: Vec<_> = (0..ROWS)
+        .map(|row| InodeLeafRow {
+            serial: first_serial + row,
+            value: encode_inode_value(InodeValue {
+                kind: InodeKind::RegularFile,
+                namespace_ref_count: 1,
+                content_root: ObjectId::for_bytes(&row.to_be_bytes()),
+                metadata_root: InodeMetadata::Inline(PortableMetadata {
+                    mode: 0o644,
+                    mtime_seconds: 1_700_000_000 + i64::from(bump),
+                    mtime_nanoseconds: bump,
+                }),
+            }),
+        })
+        .collect();
+    let canonical = InodeLeaf {
+        subtree_bytes: rows.len() as u64 * LEAF_ROW_BYTES as u64,
+        rows,
+    }
+    .encode()
+    .unwrap();
+    FinalizedObject::new(ObjectRole::InodeLeaf, canonical).unwrap()
+}
+
+/// Values that differ only in mtime low bytes and nanoseconds are distinct
+/// values: each takes one ordinal in a Save, and each one's later copies in
+/// the same Save reuse it (#91; the memo once keyed on part of the value let
+/// the first such value hide the second, whose copies then each took one).
+#[test]
+fn one_save_stores_values_that_differ_only_in_mtime_once_each() {
+    let metadata = Arc::new(metadata::MemoryMetadata::default());
+    let store = Storage::new(metadata.clone()).unwrap();
+    let save = store.begin_save().unwrap();
+    let leaves = [
+        inline_leaf(1, 0),
+        inline_leaf(100, 1),
+        inline_leaf(200, 1),
+        inline_leaf(300, 0),
+    ];
+    for object in &leaves {
+        save.accept(object.clone()).unwrap();
+    }
+    let saved = save.finish().unwrap();
+    assert_eq!(saved.pool.new_values, 2 * ROWS);
+    assert_eq!(saved.pool.reused_values, 2 * ROWS);
+    assert_eq!(metadata.state.lock().unwrap().values.len() as u64, 2 * ROWS);
+    let ids: Vec<_> = leaves.iter().map(FinalizedObject::id).collect();
+    let canonical: Vec<_> = leaves.iter().map(|l| l.canonical().to_vec()).collect();
+    assert_eq!(
+        Storage::new(metadata)
+            .unwrap()
+            .reader()
+            .unwrap()
+            .read_objects(&ids)
+            .unwrap(),
+        canonical
+    );
+}

docs/architecture/core/physical-storage/encoding.md

@@ -131,7 +131,7 @@ Inode leaves and inode-table branches have their own metadata grammar
  pooled leaf physical body:  prefix || (serial, ordinal) * rows
  record:  0 | body                                  (POOLED_FULL_TAG)
           1 | base id | out_len | count | program   (POOLED_DELTA_TAG)
- ordinal ---> inode_value row (73 bytes), stored by the Store
+ ordinal ---> inode value (73 bytes), stored by the Store (in blocks, #91)
 ```
 
 | Part | Rule | Source |

docs/architecture/engine/commit/publish-and-merge.md

@@ -106,6 +106,13 @@ manager does after removing `node_modules`. New inodes, equal bytes.
    none              --> build; record_file_digests    (digests_recorded)
 ```
 
+The Store keeps a digest only for a chunked file, at or above the Store's
+`small_file_threshold_bytes` (#91): `record_file_digests` drops smaller rows
+and `file_roots` answers none for a smaller length without a statement. A
+small file is one whole-file object, so building it again yields the same
+object id, which wave admission finds stored and does not encode again: it
+counts as built, not as `files_reused`, and stores nothing new.
+
 | Step | Source |
 | --- | --- |
 | The overlay keeps a whole-content digest for a file it followed from creation; an open file has none | [`digest.rs`](../../../../engine/crates/layerfs-overlay/src/payload/digest.rs) |

docs/architecture/engine/store/history-publication.md

@@ -38,7 +38,7 @@ The entities are defined in [Core: History model](../../core/history-model/READM
 | `commit_branch` | Check the expected head and base; insert or verify the Commit; advance the Branch head | moved head: `HeadMoved`, unless `force` with the same base; candidate equals current root: `UpToDate` | [`commit.rs`](../../../../engine/crates/layerfs-persistence/src/history/commit.rs) |
 | `add_layer` | Publish a Branch head Commit as a new Layer on the stack head; record how many Layers it overwrote | Branch moved: `HeadMoved`; stack moved: `StackMoved`; Commit base differs: `BaseMismatch`; same root: `NoChanges` | [`layerstack.rs`](../../../../engine/crates/layerfs-persistence/src/history/layerstack.rs) |
 | `reserve_inodes` | Consume a half-open inode range for a scope; advance `scope_allocator.highwater` | past `i64::MAX`: `Capacity` | [`allocation.rs`](../../../../engine/crates/layerfs-persistence/src/history/allocation.rs) |
-| `record_bindings`, `record_file_digests` | Insert name hints and file digests (array form on PostgreSQL) | — | [`hints.rs`](../../../../engine/crates/layerfs-persistence/src/history/hints.rs), [`digests.rs`](../../../../engine/crates/layerfs-persistence/src/history/digests.rs) |
+| `record_bindings`, `record_file_digests` | Append name-hint blocks; insert file digests of chunked files only, at or above `small_file_threshold_bytes` (array form on PostgreSQL) | — | [`hints.rs`](../../../../engine/crates/layerfs-persistence/src/history/hints.rs), [`digests.rs`](../../../../engine/crates/layerfs-persistence/src/history/digests.rs) |
 
 A Commit id is derived from its root, parent, base Layer, second parent and
 metadata. A Commit whose identical row exists is verified, not inserted

docs/architecture/engine/store/lifecycle.md

@@ -44,8 +44,8 @@ only by creating it ([PostgreSQL](postgresql.md)).
 | --- | --- | --- | --- |
 | `PersistenceConfig.path` | host-local file | Never a network endpoint. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
 | `sqlite_profile` | `Durable`, `Disposable` | Selected before open; never from failure or environment. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
-| `sqlite_pack_layout` | `Monolithic` (16), `GroupRows` (17), `GroupRowsIndexed` (18) | Creation only; open reads the stored version. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
-| `sqlite_acquisition` | `Absent` (+0), `Tables` (+6) | Creation only; schema versions 22–24 carry Init's tables. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
+| `sqlite_pack_layout` | `Monolithic` (36), `GroupRows` (37), `GroupRowsIndexed` (38) | Creation only; open reads the stored version. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
+| `sqlite_acquisition` | `Absent` (+0), `Tables` (+6) | Creation only; schema versions 42–44 carry Init's tables. Versions 36–44 are schema version 2 (#91, blocks); the version 1 Stores 16–18 and 22–24 are refused at open, never converted. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
 | `PostgresSelection.url` | libpq string or URL | Redacted by `Debug`. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
 | `PostgresSelection.schema` | `[a-z_][a-z0-9_]*`, at most 63 bytes, not `pg_*` | Checked by `Settings::new`. | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/session/profile.rs) |
 | `synchronous_commit` | `Off` (default), `Local`, `RemoteWrite`, `On`, `RemoteApply` | `SET LOCAL` per write. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) |
@@ -62,8 +62,8 @@ Every product caller overrides it with
 SqliteHandles::create(config, policy,     SqliteHandles::open(config, ..)
                       history)              platform is macOS or Linux?
   platform is macOS or Linux?               Session::connect(writable)
-  policy and catalog config valid?            user_version in 16..=18
-  create_new(path)                            or 22..=24, else Integrity
+  policy and catalog config valid?            user_version in 36..=38
+  create_new(path)                            or 42..=44, else Integrity
     (an existing file refuses)              writable only:
   Session::connect(create)                    add_binding_hints
   Session::initialize: schema scripts,        add_lineage (+ backfill)

docs/architecture/engine/store/metadata-and-objects.md

@@ -62,20 +62,50 @@ signature ring is an upsert by slot that keeps the newer stamp
 ## Values
 
 ```text
- inode_value
- +-----------+------------------------------+
- | ordinal   | value (73 bytes)             |   one row per ordinal,
- | 1..2^32-1 |                              |   INSERT only, never updated
- +-----------+------------------------------+
- read: ascending distinct ordinals -> JSON runs [[first,last],..]
-       -> one statement: json_each JOIN inode_value BETWEEN first AND last
+ inode_value_block (#91, schema version 2)
+ +---------------+--------------+------------------------------------+
+ | first_ordinal | last_ordinal | body (<= 4,000 bytes, PLAIN)       |  one row per block of
+ | 1..2^32-1     | first+0..254 | values of first..=last, CRC32C     |  consecutive ordinals,
+ +---------------+--------------+------------------------------------+  INSERT only
+ read: ascending distinct ordinals -> runs [[first,last],..]
+       -> one statement: per run, the last block starting at or before it
+          and every block from there to its end; decode, pick asked ordinals
 ```
 
 `inode_values(ordinals)` requires ascending, distinct, nonzero ordinals (at
-most 4 096) and refuses a reply that names an ordinal not asked. A second
-INSERT of an ordinal fails its transaction. Sources:
+most 4 096) and refuses a reply that names an ordinal not asked. A block whose
+ordinals a stored block already holds is not inserted, and the publication
+fails: a stored value is never replaced. Sources:
 [`values.rs`](../../../../engine/crates/layerfs-persistence/src/metadata/values.rs),
-[`sqlite/metadata/values.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs).
+[`sqlite/metadata/values.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs),
+[`postgresql/metadata/reads.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs).
+
+### Inode value and binding hint blocks
+
+Schema version 2 (#91) keeps inode values and binding hints in append-only,
+immutable blocks instead of one row each, so a database row's fixed cost
+(SQLite cell and index entry; PostgreSQL's 28-byte tuple header and line
+pointer plus a B-tree entry) is paid per block of about 100 entries. The
+encoding is field packing only, with no compression: each entry expands back
+to its exact canonical bytes, so no object identity depends on it
+([`blocks/`](../../../../engine/crates/layerfs-persistence/src/blocks/mod.rs)).
+Every body ends with a CRC32C of the bytes before it, checked on every
+decode; a block that fails it fails the read.
+
+| Block | Key | Body |
+| --- | --- | --- |
+| Inode values | `first_ordinal`; ordinals inside implicit (`last_ordinal` kept for the overlap check) | format, count, a table of the distinct mode words, then per entry the value's byte 0 as tag, a LEB128 reference count, the 32-byte content root and either the 32-byte attribute root (root form) or a mode-table index, the mtime seconds as a zig-zag LEB128 difference from the previous inline entry and the four nanosecond bytes (inline form); any other 73 bytes are stored whole under tag `0xff` ([`values.rs`](../../../../engine/crates/layerfs-persistence/src/blocks/values.rs)) |
+| Binding hints | `(grp, seq)`: `grp = child >> 6`, `seq` the next of its group | format, LEB128 count, then per entry in `(child, parent, name)` order the child's LEB128 difference from the previous entry's, the parent's zig-zag LEB128 difference, the shared name-prefix length and the rest of the name ([`hints.rs`](../../../../engine/crates/layerfs-persistence/src/blocks/hints.rs)) |
+
+A Save's publication encodes its values into blocks before its statement:
+a new block starts at a gap in the ordinals or when the current one would pass
+4,000 bytes or 255 entries. One write of binding hints appends one or more
+blocks per group it touches, numbered under the write lane; a reader takes
+every block of the child's group, merges them and drops repeats (a binding
+recorded twice is kept twice on disk). Rows are never updated or deleted
+(gap design R6). The diagnostic decoder
+[`tools/store_blocks.py`](../../../../tools/store_blocks.py) reads both
+from a closed SQLite copy or from exported PostgreSQL rows for attribution.
 
 ## Object reads
 
@@ -85,7 +115,7 @@ INSERT of an ordinal fails its transaction. Sources:
 | `read_pack_selection(id, plan)` | `read_immutable` | exact offset and length BLOB reads planned by core | [`selection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/objects/selection.rs) |
 | `read_scoped_pack(id, plan)` | `read_immutable` | as above, for acquisition | same |
 
-On layouts 17 and 18 a pack is a `control` row plus one `pack_unit` row per
+On layouts 37 and 38 a pack is a `control` row plus one `pack_unit` row per
 group, so a selected group is read without the rest of the pack
 ([`units_read.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/objects/units_read.rs)).
 PostgreSQL reads ranges of the `EXTERNAL` body with `substring`
@@ -114,7 +144,8 @@ visible ([SQLite: Sessions](sqlite.md#sessions)).
      INSERT pack (or control + pack_unit rows)
      INSERT object_location ... ON CONFLICT DO NOTHING RETURNING
      UPDATE store_totals (only when bytes were added)
-     INSERT inode_value
+     INSERT inode_value_block ... WHERE no stored block overlaps
+       RETURNING (a missing row fails the publication)
      give back the ordinal tail (release_ordinals)
      UPSERT content_signature
    -> Published { lost }   objects another publication registered first

docs/architecture/engine/store/postgresql.md

@@ -23,10 +23,10 @@ Without it, selecting a PostgreSQL Store fails `BackendUnavailable`.
  +--------------------------------------------------------------+
  | schema <selection.schema>      (one Store; [a-z_][a-z0-9_]*) |
  |  store_lane (id=1)   <-- the row every writer locks          |
- |  store_policy, pack, object_location, inode_value,           |
+ |  store_policy, pack, object_location, inode_value_block,     |
  |  content_signature, store_totals                             |
  |  history_meta, layer_stack, layer, "commit", branch,         |
- |  scope_allocator, binding_hint                               |
+ |  scope_allocator, binding_hint_block                         |
  |  layer_lineage, commit_lineage, absorbed, reach, file_digest |
  |  init_operation, init_entry, init_host_file   (if created)   |
  +--------------------------------------------------------------+
@@ -40,10 +40,13 @@ Without it, selecting a PostgreSQL Store fails `BackendUnavailable`.
 | Types | `bigint` for integers, `bytea` for BLOBs | [`schema.sql`](../../../../engine/crates/layerfs-persistence/sql/postgresql/schema.sql) |
 | Name order | history names `COLLATE "C"`, so paging matches SQLite byte order | same |
 | Pack bodies | one `bytea` per pack, `STORAGE EXTERNAL` (uncompressed, out of line) for ranged `substring` reads | same |
-| Pack units | absent: group rows are a SQLite physical layout | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) |
+| Pack units | absent: group rows are a SQLite physical layout. Explicit 8,100-byte `PLAIN` chunk rows were measured and not taken (#90): one chunk per page and a partly filled page per pack tail made D10's packs 44.0 MB, against 43.2 MB with `EXTERNAL` |
+| Object locations | narrow rows (#90): `pack_id bigint`, `canonical_length integer`, `role`, `group_number`, `record_number` `smallint`, then `object_id bytea STORAGE PLAIN`; no unique key on the id but a B-tree `object_location_prefix` on its first eight bytes as a signed big-endian `bigint`, with the full id compared in the heap. A location is inserted only when its id is absent (`INSERT … WHERE NOT EXISTS`), which is unique because every location insert holds the lane; the lost-race and Collision read-back are unchanged. A signature must name a location of its publication or a stored one, checked in the same statement (the foreign key needed a unique key). D10: 9.70 → 5.82 MB | [`publish.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/objects/publish.rs), [`reads.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs) |
+| No TOAST for short values | every bounded `bytea`/`text` column of `object_location`, `content_signature`, `file_digest`, `layer_stack`, `branch`, `scope_allocator`, `layer_lineage`, `commit_lineage`, `absorbed` and the acquisition tables is `STORAGE PLAIN`, so those tables have no TOAST relation (8 KiB each); `pack`, `layer`, `commit`, `history_meta` and `reach` keep one for their unbounded columns | [`schema.sql`](../../../../engine/crates/layerfs-persistence/sql/postgresql/schema.sql) |
+| Blocks | `inode_value_block.body` and `binding_hint_block.body` are `bytea STORAGE PLAIN` in the `CREATE TABLE` (PostgreSQL 16 or newer): two 4,000-byte rows per 8 KiB page, never TOASTed or compressed, and no TOAST table | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) |
 | Lane | `store_lane`, one row | [`schema.sql`](../../../../engine/crates/layerfs-persistence/sql/postgresql/schema.sql) |
 | Merge records | temporary tables of the writer connection | [`merge_records.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/merge_records.rs) |
-| Schema version | `SCHEMA_VERSION = 1`, engine-scoped | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) |
+| Schema version | `SCHEMA_VERSION = 2` (#91: inode value and binding hint blocks; #90: narrow locations, no short-value TOAST), engine-scoped; a version 1 schema is refused at open (`Integrity`), never converted | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) |
 
 `PostgresHandles::create` checks `pg_namespace` and refuses an existing
 schema (`Refused "schema exists"`) before writing. It then creates the