diff --git a/core/crates/layerfs-storage/README.md b/core/crates/layerfs-storage/README.md index 7912c09a83..0daa9bc3fd 100644 --- a/core/crates/layerfs-storage/README.md +++ b/core/crates/layerfs-storage/README.md @@ -120,7 +120,8 @@ destructor. - Filesystem trees, runtime adapters and cloud placement are later scope. Payload DELTA chains and pooled metadata are implemented. -- Inode values are rows of `inode_value` (ordinal, 73 value bytes). The earlier +- Inode values are stored per ordinal through `PackPersistence` (73 value bytes); + the engine keeps them in append-only blocks since its schema version 2 (#91). The earlier value-group catalogue and v6 value packs are no longer written or read; a Store that has them is refused at open. - The retained index window boundary (131,072 entries) and its cold-start replay diff --git a/core/crates/layerfs-storage/src/save/pooled.rs b/core/crates/layerfs-storage/src/save/pooled.rs index b19b4b09b5..f5b01ec28a 100644 --- a/core/crates/layerfs-storage/src/save/pooled.rs +++ b/core/crates/layerfs-storage/src/save/pooled.rs @@ -33,14 +33,20 @@ struct Base { chain_encoded: u64, } -/// Candidate filter of one value: eight bytes of each of its two digests. +/// Candidate filter of one value: a 64-bit mix of all 73 bytes. A filter of +/// only some bytes let two values that differ elsewhere (the same content and +/// mode with another mtime) share one memo entry, so the second one's later +/// copies in the same Save each took a new ordinal (#91). fn fingerprint(value: &[u8; INODE_VALUE_BYTES]) -> u64 { - let word = |from: usize| { - let mut bytes = [0_u8; 8]; - bytes.copy_from_slice(&value[from..from + 8]); - u64::from_le_bytes(bytes) - }; - word(INODE_VALUE_BYTES - 64) ^ word(INODE_VALUE_BYTES - 32).rotate_left(29) ^ word(0) + value + .chunks(8) + .fold(0x9e37_79b9_7f4a_7c15_u64, |hash, chunk| { + let mut word = [0_u8; 8]; + word[..chunk.len()].copy_from_slice(chunk); + (hash ^ u64::from_le_bytes(word)) + .wrapping_mul(0x0000_0100_0000_01b3) + .rotate_left(29) + }) } impl State<'_> { diff --git a/core/crates/layerfs-storage/tests/stored_value_reads.rs b/core/crates/layerfs-storage/tests/stored_value_reads.rs index 3c26bb7918..097c47086f 100644 --- a/core/crates/layerfs-storage/tests/stored_value_reads.rs +++ b/core/crates/layerfs-storage/tests/stored_value_reads.rs @@ -257,3 +257,68 @@ fn one_handle_reads_any_number_of_leaves_at_one_lookup_each() { } assert_eq!(value_lookups(&metadata) as u64, LEAVES); } + +/// A leaf of `ROWS` files of one content and mode whose mtime differs from +/// `mtime` only in its low bytes and nanoseconds by `bump`. +fn inline_leaf(first_serial: u64, bump: u32) -> FinalizedObject { + use layerfs_content::{ + filesystem::attributes::portable::PortableMetadata, inode_leaf::InodeMetadata, + }; + let rows: Vec<_> = (0..ROWS) + .map(|row| InodeLeafRow { + serial: first_serial + row, + value: encode_inode_value(InodeValue { + kind: InodeKind::RegularFile, + namespace_ref_count: 1, + content_root: ObjectId::for_bytes(&row.to_be_bytes()), + metadata_root: InodeMetadata::Inline(PortableMetadata { + mode: 0o644, + mtime_seconds: 1_700_000_000 + i64::from(bump), + mtime_nanoseconds: bump, + }), + }), + }) + .collect(); + let canonical = InodeLeaf { + subtree_bytes: rows.len() as u64 * LEAF_ROW_BYTES as u64, + rows, + } + .encode() + .unwrap(); + FinalizedObject::new(ObjectRole::InodeLeaf, canonical).unwrap() +} + +/// Values that differ only in mtime low bytes and nanoseconds are distinct +/// values: each takes one ordinal in a Save, and each one's later copies in +/// the same Save reuse it (#91; the memo once keyed on part of the value let +/// the first such value hide the second, whose copies then each took one). +#[test] +fn one_save_stores_values_that_differ_only_in_mtime_once_each() { + let metadata = Arc::new(metadata::MemoryMetadata::default()); + let store = Storage::new(metadata.clone()).unwrap(); + let save = store.begin_save().unwrap(); + let leaves = [ + inline_leaf(1, 0), + inline_leaf(100, 1), + inline_leaf(200, 1), + inline_leaf(300, 0), + ]; + for object in &leaves { + save.accept(object.clone()).unwrap(); + } + let saved = save.finish().unwrap(); + assert_eq!(saved.pool.new_values, 2 * ROWS); + assert_eq!(saved.pool.reused_values, 2 * ROWS); + assert_eq!(metadata.state.lock().unwrap().values.len() as u64, 2 * ROWS); + let ids: Vec<_> = leaves.iter().map(FinalizedObject::id).collect(); + let canonical: Vec<_> = leaves.iter().map(|l| l.canonical().to_vec()).collect(); + assert_eq!( + Storage::new(metadata) + .unwrap() + .reader() + .unwrap() + .read_objects(&ids) + .unwrap(), + canonical + ); +} diff --git a/docs/architecture/core/physical-storage/encoding.md b/docs/architecture/core/physical-storage/encoding.md index 5a7ac75647..520d98a7a1 100644 --- a/docs/architecture/core/physical-storage/encoding.md +++ b/docs/architecture/core/physical-storage/encoding.md @@ -131,7 +131,7 @@ Inode leaves and inode-table branches have their own metadata grammar pooled leaf physical body: prefix || (serial, ordinal) * rows record: 0 | body (POOLED_FULL_TAG) 1 | base id | out_len | count | program (POOLED_DELTA_TAG) - ordinal ---> inode_value row (73 bytes), stored by the Store + ordinal ---> inode value (73 bytes), stored by the Store (in blocks, #91) ``` | Part | Rule | Source | diff --git a/docs/architecture/engine/commit/publish-and-merge.md b/docs/architecture/engine/commit/publish-and-merge.md index 5589054330..ec80aa53c5 100644 --- a/docs/architecture/engine/commit/publish-and-merge.md +++ b/docs/architecture/engine/commit/publish-and-merge.md @@ -106,6 +106,13 @@ manager does after removing `node_modules`. New inodes, equal bytes. none --> build; record_file_digests (digests_recorded) ``` +The Store keeps a digest only for a chunked file, at or above the Store's +`small_file_threshold_bytes` (#91): `record_file_digests` drops smaller rows +and `file_roots` answers none for a smaller length without a statement. A +small file is one whole-file object, so building it again yields the same +object id, which wave admission finds stored and does not encode again: it +counts as built, not as `files_reused`, and stores nothing new. + | Step | Source | | --- | --- | | The overlay keeps a whole-content digest for a file it followed from creation; an open file has none | [`digest.rs`](../../../../engine/crates/layerfs-overlay/src/payload/digest.rs) | diff --git a/docs/architecture/engine/store/history-publication.md b/docs/architecture/engine/store/history-publication.md index b6d708b661..a4147a1e60 100644 --- a/docs/architecture/engine/store/history-publication.md +++ b/docs/architecture/engine/store/history-publication.md @@ -38,7 +38,7 @@ The entities are defined in [Core: History model](../../core/history-model/READM | `commit_branch` | Check the expected head and base; insert or verify the Commit; advance the Branch head | moved head: `HeadMoved`, unless `force` with the same base; candidate equals current root: `UpToDate` | [`commit.rs`](../../../../engine/crates/layerfs-persistence/src/history/commit.rs) | | `add_layer` | Publish a Branch head Commit as a new Layer on the stack head; record how many Layers it overwrote | Branch moved: `HeadMoved`; stack moved: `StackMoved`; Commit base differs: `BaseMismatch`; same root: `NoChanges` | [`layerstack.rs`](../../../../engine/crates/layerfs-persistence/src/history/layerstack.rs) | | `reserve_inodes` | Consume a half-open inode range for a scope; advance `scope_allocator.highwater` | past `i64::MAX`: `Capacity` | [`allocation.rs`](../../../../engine/crates/layerfs-persistence/src/history/allocation.rs) | -| `record_bindings`, `record_file_digests` | Insert name hints and file digests (array form on PostgreSQL) | — | [`hints.rs`](../../../../engine/crates/layerfs-persistence/src/history/hints.rs), [`digests.rs`](../../../../engine/crates/layerfs-persistence/src/history/digests.rs) | +| `record_bindings`, `record_file_digests` | Append name-hint blocks; insert file digests of chunked files only, at or above `small_file_threshold_bytes` (array form on PostgreSQL) | — | [`hints.rs`](../../../../engine/crates/layerfs-persistence/src/history/hints.rs), [`digests.rs`](../../../../engine/crates/layerfs-persistence/src/history/digests.rs) | A Commit id is derived from its root, parent, base Layer, second parent and metadata. A Commit whose identical row exists is verified, not inserted diff --git a/docs/architecture/engine/store/lifecycle.md b/docs/architecture/engine/store/lifecycle.md index 901114035b..d613efef34 100644 --- a/docs/architecture/engine/store/lifecycle.md +++ b/docs/architecture/engine/store/lifecycle.md @@ -44,8 +44,8 @@ only by creating it ([PostgreSQL](postgresql.md)). | --- | --- | --- | --- | | `PersistenceConfig.path` | host-local file | Never a network endpoint. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | | `sqlite_profile` | `Durable`, `Disposable` | Selected before open; never from failure or environment. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | -| `sqlite_pack_layout` | `Monolithic` (16), `GroupRows` (17), `GroupRowsIndexed` (18) | Creation only; open reads the stored version. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | -| `sqlite_acquisition` | `Absent` (+0), `Tables` (+6) | Creation only; schema versions 22–24 carry Init's tables. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | +| `sqlite_pack_layout` | `Monolithic` (36), `GroupRows` (37), `GroupRowsIndexed` (38) | Creation only; open reads the stored version. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | +| `sqlite_acquisition` | `Absent` (+0), `Tables` (+6) | Creation only; schema versions 42–44 carry Init's tables. Versions 36–44 are schema version 2 (#91, blocks); the version 1 Stores 16–18 and 22–24 are refused at open, never converted. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | | `PostgresSelection.url` | libpq string or URL | Redacted by `Debug`. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | | `PostgresSelection.schema` | `[a-z_][a-z0-9_]*`, at most 63 bytes, not `pg_*` | Checked by `Settings::new`. | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/session/profile.rs) | | `synchronous_commit` | `Off` (default), `Local`, `RemoteWrite`, `On`, `RemoteApply` | `SET LOCAL` per write. | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | @@ -62,8 +62,8 @@ Every product caller overrides it with SqliteHandles::create(config, policy, SqliteHandles::open(config, ..) history) platform is macOS or Linux? platform is macOS or Linux? Session::connect(writable) - policy and catalog config valid? user_version in 16..=18 - create_new(path) or 22..=24, else Integrity + policy and catalog config valid? user_version in 36..=38 + create_new(path) or 42..=44, else Integrity (an existing file refuses) writable only: Session::connect(create) add_binding_hints Session::initialize: schema scripts, add_lineage (+ backfill) diff --git a/docs/architecture/engine/store/metadata-and-objects.md b/docs/architecture/engine/store/metadata-and-objects.md index 08770f2912..4d8e1c4b55 100644 --- a/docs/architecture/engine/store/metadata-and-objects.md +++ b/docs/architecture/engine/store/metadata-and-objects.md @@ -62,20 +62,50 @@ signature ring is an upsert by slot that keeps the newer stamp ## Values ```text - inode_value - +-----------+------------------------------+ - | ordinal | value (73 bytes) | one row per ordinal, - | 1..2^32-1 | | INSERT only, never updated - +-----------+------------------------------+ - read: ascending distinct ordinals -> JSON runs [[first,last],..] - -> one statement: json_each JOIN inode_value BETWEEN first AND last + inode_value_block (#91, schema version 2) + +---------------+--------------+------------------------------------+ + | first_ordinal | last_ordinal | body (<= 4,000 bytes, PLAIN) | one row per block of + | 1..2^32-1 | first+0..254 | values of first..=last, CRC32C | consecutive ordinals, + +---------------+--------------+------------------------------------+ INSERT only + read: ascending distinct ordinals -> runs [[first,last],..] + -> one statement: per run, the last block starting at or before it + and every block from there to its end; decode, pick asked ordinals ``` `inode_values(ordinals)` requires ascending, distinct, nonzero ordinals (at -most 4 096) and refuses a reply that names an ordinal not asked. A second -INSERT of an ordinal fails its transaction. Sources: +most 4 096) and refuses a reply that names an ordinal not asked. A block whose +ordinals a stored block already holds is not inserted, and the publication +fails: a stored value is never replaced. Sources: [`values.rs`](../../../../engine/crates/layerfs-persistence/src/metadata/values.rs), -[`sqlite/metadata/values.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs). +[`sqlite/metadata/values.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs), +[`postgresql/metadata/reads.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs). + +### Inode value and binding hint blocks + +Schema version 2 (#91) keeps inode values and binding hints in append-only, +immutable blocks instead of one row each, so a database row's fixed cost +(SQLite cell and index entry; PostgreSQL's 28-byte tuple header and line +pointer plus a B-tree entry) is paid per block of about 100 entries. The +encoding is field packing only, with no compression: each entry expands back +to its exact canonical bytes, so no object identity depends on it +([`blocks/`](../../../../engine/crates/layerfs-persistence/src/blocks/mod.rs)). +Every body ends with a CRC32C of the bytes before it, checked on every +decode; a block that fails it fails the read. + +| Block | Key | Body | +| --- | --- | --- | +| Inode values | `first_ordinal`; ordinals inside implicit (`last_ordinal` kept for the overlap check) | format, count, a table of the distinct mode words, then per entry the value's byte 0 as tag, a LEB128 reference count, the 32-byte content root and either the 32-byte attribute root (root form) or a mode-table index, the mtime seconds as a zig-zag LEB128 difference from the previous inline entry and the four nanosecond bytes (inline form); any other 73 bytes are stored whole under tag `0xff` ([`values.rs`](../../../../engine/crates/layerfs-persistence/src/blocks/values.rs)) | +| Binding hints | `(grp, seq)`: `grp = child >> 6`, `seq` the next of its group | format, LEB128 count, then per entry in `(child, parent, name)` order the child's LEB128 difference from the previous entry's, the parent's zig-zag LEB128 difference, the shared name-prefix length and the rest of the name ([`hints.rs`](../../../../engine/crates/layerfs-persistence/src/blocks/hints.rs)) | + +A Save's publication encodes its values into blocks before its statement: +a new block starts at a gap in the ordinals or when the current one would pass +4,000 bytes or 255 entries. One write of binding hints appends one or more +blocks per group it touches, numbered under the write lane; a reader takes +every block of the child's group, merges them and drops repeats (a binding +recorded twice is kept twice on disk). Rows are never updated or deleted +(gap design R6). The diagnostic decoder +[`tools/store_blocks.py`](../../../../tools/store_blocks.py) reads both +from a closed SQLite copy or from exported PostgreSQL rows for attribution. ## Object reads @@ -85,7 +115,7 @@ INSERT of an ordinal fails its transaction. Sources: | `read_pack_selection(id, plan)` | `read_immutable` | exact offset and length BLOB reads planned by core | [`selection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/objects/selection.rs) | | `read_scoped_pack(id, plan)` | `read_immutable` | as above, for acquisition | same | -On layouts 17 and 18 a pack is a `control` row plus one `pack_unit` row per +On layouts 37 and 38 a pack is a `control` row plus one `pack_unit` row per group, so a selected group is read without the rest of the pack ([`units_read.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/objects/units_read.rs)). PostgreSQL reads ranges of the `EXTERNAL` body with `substring` @@ -114,7 +144,8 @@ visible ([SQLite: Sessions](sqlite.md#sessions)). INSERT pack (or control + pack_unit rows) INSERT object_location ... ON CONFLICT DO NOTHING RETURNING UPDATE store_totals (only when bytes were added) - INSERT inode_value + INSERT inode_value_block ... WHERE no stored block overlaps + RETURNING (a missing row fails the publication) give back the ordinal tail (release_ordinals) UPSERT content_signature -> Published { lost } objects another publication registered first diff --git a/docs/architecture/engine/store/postgresql.md b/docs/architecture/engine/store/postgresql.md index 77cd067ec4..b40d1698c8 100644 --- a/docs/architecture/engine/store/postgresql.md +++ b/docs/architecture/engine/store/postgresql.md @@ -23,10 +23,10 @@ Without it, selecting a PostgreSQL Store fails `BackendUnavailable`. +--------------------------------------------------------------+ | schema (one Store; [a-z_][a-z0-9_]*) | | store_lane (id=1) <-- the row every writer locks | - | store_policy, pack, object_location, inode_value, | + | store_policy, pack, object_location, inode_value_block, | | content_signature, store_totals | | history_meta, layer_stack, layer, "commit", branch, | - | scope_allocator, binding_hint | + | scope_allocator, binding_hint_block | | layer_lineage, commit_lineage, absorbed, reach, file_digest | | init_operation, init_entry, init_host_file (if created) | +--------------------------------------------------------------+ @@ -40,10 +40,13 @@ Without it, selecting a PostgreSQL Store fails `BackendUnavailable`. | Types | `bigint` for integers, `bytea` for BLOBs | [`schema.sql`](../../../../engine/crates/layerfs-persistence/sql/postgresql/schema.sql) | | Name order | history names `COLLATE "C"`, so paging matches SQLite byte order | same | | Pack bodies | one `bytea` per pack, `STORAGE EXTERNAL` (uncompressed, out of line) for ranged `substring` reads | same | -| Pack units | absent: group rows are a SQLite physical layout | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) | +| Pack units | absent: group rows are a SQLite physical layout. Explicit 8,100-byte `PLAIN` chunk rows were measured and not taken (#90): one chunk per page and a partly filled page per pack tail made D10's packs 44.0 MB, against 43.2 MB with `EXTERNAL` | +| Object locations | narrow rows (#90): `pack_id bigint`, `canonical_length integer`, `role`, `group_number`, `record_number` `smallint`, then `object_id bytea STORAGE PLAIN`; no unique key on the id but a B-tree `object_location_prefix` on its first eight bytes as a signed big-endian `bigint`, with the full id compared in the heap. A location is inserted only when its id is absent (`INSERT … WHERE NOT EXISTS`), which is unique because every location insert holds the lane; the lost-race and Collision read-back are unchanged. A signature must name a location of its publication or a stored one, checked in the same statement (the foreign key needed a unique key). D10: 9.70 → 5.82 MB | [`publish.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/objects/publish.rs), [`reads.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs) | +| No TOAST for short values | every bounded `bytea`/`text` column of `object_location`, `content_signature`, `file_digest`, `layer_stack`, `branch`, `scope_allocator`, `layer_lineage`, `commit_lineage`, `absorbed` and the acquisition tables is `STORAGE PLAIN`, so those tables have no TOAST relation (8 KiB each); `pack`, `layer`, `commit`, `history_meta` and `reach` keep one for their unbounded columns | [`schema.sql`](../../../../engine/crates/layerfs-persistence/sql/postgresql/schema.sql) | +| Blocks | `inode_value_block.body` and `binding_hint_block.body` are `bytea STORAGE PLAIN` in the `CREATE TABLE` (PostgreSQL 16 or newer): two 4,000-byte rows per 8 KiB page, never TOASTed or compressed, and no TOAST table | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) | | Lane | `store_lane`, one row | [`schema.sql`](../../../../engine/crates/layerfs-persistence/sql/postgresql/schema.sql) | | Merge records | temporary tables of the writer connection | [`merge_records.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/merge_records.rs) | -| Schema version | `SCHEMA_VERSION = 1`, engine-scoped | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) | +| Schema version | `SCHEMA_VERSION = 2` (#91: inode value and binding hint blocks; #90: narrow locations, no short-value TOAST), engine-scoped; a version 1 schema is refused at open (`Integrity`), never converted | [`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) | `PostgresHandles::create` checks `pg_namespace` and refuses an existing schema (`Refused "schema exists"`) before writing. It then creates the diff --git a/docs/architecture/engine/store/sqlite.md b/docs/architecture/engine/store/sqlite.md index a636545707..bf2a745648 100644 --- a/docs/architecture/engine/store/sqlite.md +++ b/docs/architecture/engine/store/sqlite.md @@ -25,41 +25,41 @@ The SQLite backend. | object_id PK | | +----------------+ | +--------------------+ v v +--------------------+ +-------------+ +----------------+ +-------------+ - | inode_value | | pack | | branch | | layer | - | ordinal PK | | pack_id PK | | base_layer_id -+>| layer_id PK | + | inode_value_block | | pack | | branch | | layer | + | first_ordinal PK | | pack_id PK | | base_layer_id -+>| layer_id PK | +--------------------+ +-------------+ | head_commit_id | +-------------+ +--------------------+ ^ +-------+--------+ ^ | content_signature | | v | | slot PK 0..8191 | +-------------+ +----------------+ | +--------------------+ | pack_unit | | "commit" | | - +--------------------+ | (17 and 18) | | commit_id PK | | + +--------------------+ | (37 and 38) | | commit_id PK | | | store_totals id=1 | +-------------+ | base_layer_id -+--------+ +--------------------+ +----------------+ lineage: layer_lineage, commit_lineage, absorbed, reach - other: scope_allocator, binding_hint, file_digest, + other: scope_allocator, binding_hint_block, file_digest, merge_operation, merge_record - Init: init_operation, init_entry, init_host_file (versions 22-24) + Init: init_operation, init_entry, init_host_file (versions 42-44) ``` | Table | Key | Holds | File | | --- | --- | --- | --- | | `store_policy` | `id=1` | Frozen policy, schema version, `next_pack_id`, `next_ordinal` | [`metadata.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/metadata.sql) | | `object_location` | `object_id` (32 bytes) | Role, canonical length, pack, group and record number | same | -| `inode_value` | `ordinal` | One 73-byte stored inode value | same | +| `inode_value_block` | `first_ordinal` | The 73-byte inode values of ordinals `first_ordinal..=last_ordinal` (at most 255) in one block of at most 4,000 bytes (#91; [blocks](metadata-and-objects.md#inode-value-and-binding-hint-blocks)) | same | | `content_signature` | `slot` 0..8191 | Ring of content signatures, ordered by `stamp` | same | | `store_totals` | `id=1` | Logical, canonical, stored, metadata and Init totals | same | -| `pack` | `pack_id` | Layout 16: whole `body`; 17–18: `control` only; immutable by trigger | [`objects.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects.sql), [`objects_units.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects_units.sql) | -| `pack_unit` | `unit_id`; unique `(pack_id, group_number)` | One encoded group per row (layouts 17–18; 18 adds index `pack_unit_mapping`) | [`objects_units.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects_units.sql) | +| `pack` | `pack_id` | Layout 36: whole `body`; 37–38: `control` only; immutable by trigger | [`objects.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects.sql), [`objects_units.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects_units.sql) | +| `pack_unit` | `unit_id`; unique `(pack_id, group_number)` | One encoded group per row (layouts 37–38; 38 adds index `pack_unit_mapping`) | [`objects_units.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects_units.sql) | | `history_meta` | `id=1` | Catalog id, incarnation, binding key, recorded schema source and definition | [`history.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/history.sql) | | `layer_stack` | `layer_stack_id`; unique `name` | A project and its head Layer | same | | `layer` | `layer_id` | Root, parent Layer, source Branch and Commit, `logical` | same | | `"commit"` | `commit_id` | Root, parent, base Layer, second parent, Branch, message, statistics, LWW facts, displaced head | same | | `branch` | `branch_id`; unique `(layer_stack_id, name)` | Base Layer, `forked_from_id`, head Commit | same | | `scope_allocator` | `scope_id` | Inode high-water per scope | same | -| `binding_hint` | `(child, parent, name)` | Name bindings of saved trees | same | +| `binding_hint_block` | `(grp, seq)` | One write's name bindings of saved trees for the 64 serials of group `grp` (`child >> 6`), in a block of at most 4,000 bytes (#91) | same | | `layer_lineage`, `commit_lineage`, `absorbed`, `reach` | see file | Ancestry without walking history | [`lineage.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/lineage.sql) | | `merge_operation`, `merge_record` | `operation`; `(operation, kind, key)` | One merge's spilled build records | [`merge_scratch.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/merge_scratch.sql) | -| `file_digest` | `(digest, length)` | Saved file root by content digest | [`file_digest.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/file_digest.sql) | +| `file_digest` | `(digest, length)` | Saved file root by content digest, chunked files only (length at or above `small_file_threshold_bytes`) | [`file_digest.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/file_digest.sql) | | `init_*` | operation-prefixed | Init's acquisition working state | [`acquisition/schema.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/acquisition/schema.sql) | Every table is `STRICT`; history keys are `WITHOUT ROWID` blobs whose first @@ -76,7 +76,7 @@ text ([`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sq | Open | `READ_WRITE` or `READ_ONLY`, `NO_MUTEX`; statement cache 64 | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Guard | `busy_timeout=0`; `SQLITE_DBCONFIG_DEFENSIVE` on | same | | Settings | `profile::apply`, then read back and `profile::check` | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | -| Version | `user_version` 16–18 or 22–24; anything else `Integrity` | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | +| Version | `user_version` 36–38 or 42–44 (schema version 2, #91); anything else, including the schema version 1 Stores 16–18 and 22–24, `Integrity` | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Call | Transaction | Takes the writer lock | Source | | --- | --- | --- | --- | @@ -87,7 +87,9 @@ text ([`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sq `read_immutable` is limited to rows that never change once visible: `object_location` (insert-only), `pack` and `pack_unit` (triggers refuse -UPDATE and DELETE), and `inode_value` (one INSERT per ordinal). Anything that +UPDATE and DELETE), and `inode_value_block` (append-only blocks; an INSERT of +a block whose ordinals a stored block holds inserts nothing and fails the +publication). Anything that picks a snapshot, such as Branch heads or Commit rows, uses `run`. Each turn also adds its SQL work to the calling thread's counters ([`thread_work.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/thread_work.rs)). diff --git a/engine/crates/layerfs-daemon/tests/file_digests.rs b/engine/crates/layerfs-daemon/tests/file_digests.rs index 7c58e3009f..76c268d1db 100644 --- a/engine/crates/layerfs-daemon/tests/file_digests.rs +++ b/engine/crates/layerfs-daemon/tests/file_digests.rs @@ -1,7 +1,10 @@ //! #364 item 2 (b): a captured Commit records the Store digest of each fresh //! file the overlay followed from creation, and a fresh file whose digest and //! length a saved root already has reuses that object: its own row, serial -//! and metadata stay (owner decision, object reuse only). +//! and metadata stay (owner decision, object reuse only). The Store keeps +//! digests only of chunked files, at or above the small-file threshold +//! (#91): a smaller file is one whole-file object, which a rebuild finds by +//! its identity, so it is built again and stores nothing new. #[path = "support/installed_store.rs"] #[allow(dead_code)] mod support; @@ -158,6 +161,8 @@ fn commit(workspace: &BoundWorkspace) -> (CommitSuccess, CapturedNamespaceWork) let work = done.namespace.as_ref().unwrap().work.unwrap(); (done, work) } +/// The frozen default small-file threshold: the first chunked length. +const CHUNKED: usize = 131_072; fn bytes(tag: u8, length: usize) -> Vec { (0..length).map(|at| tag ^ (at % 251) as u8).collect() } @@ -216,30 +221,38 @@ impl Setup { #[test] fn a_commit_records_digests_of_followed_fresh_files_only() { let s = setup("file-digests"); - put(&s.workspace, "appended", 0o644, b"hello digest"); + let large = bytes(6, CHUNKED); + put(&s.workspace, "appended", 0o644, &large); + put(&s.workspace, "small", 0o644, b"hello digest"); // Rewritten at offset 0 after its first bytes: not followed. - let rewritten = create(&s.workspace, "rewritten", 0o644); - write(&s.workspace, rewritten, 0, b"other bytes"); + let mut other = bytes(7, CHUNKED + 1); + let rewritten = put(&s.workspace, "rewritten", 0o644, &other); write(&s.workspace, rewritten, 0, b"O"); + other[0] = b'O'; let (done, work) = commit(&s.workspace); - assert_eq!(work.digests_recorded, 1); + // Both followed files are offered; the Store keeps the chunked one. + assert_eq!(work.digests_recorded, 2); let appended = committed(&s.store, &s.workspace, "appended").1.content_root; let digest = |bytes: &[u8]| *blake3::hash(bytes).as_bytes(); let roots = s .store .history() - .file_roots(&[(digest(b"hello digest"), 12), (digest(b"Other bytes"), 11)]) + .file_roots(&[ + (digest(&large), large.len() as u64), + (digest(&other), other.len() as u64), + (digest(b"hello digest"), 12), + ]) .unwrap(); - assert_eq!(roots[0], Some(appended.to_bytes())); - assert_eq!(roots[1], None); + assert_eq!(roots, vec![Some(appended.to_bytes()), None, None]); drop(done); s.finish(); } /// A clean relink over files a Commit built: the directory and every file -/// are removed and written again, and one copy lands at a new path. A file -/// whose bytes a saved root holds reuses that object and keeps its own -/// serial and metadata; every other file takes the full path. +/// are removed and written again, and one copy lands at a new path. A +/// chunked file whose bytes a saved root holds reuses that object and keeps +/// its own serial and metadata; every other file takes the full path, where +/// a small file with the same bytes builds the same object again. #[test] fn a_relinked_file_with_saved_bytes_reuses_the_object() { let s = setup("file-digests-relink"); @@ -247,7 +260,7 @@ fn a_relinked_file_with_saved_bytes_reuses_the_object() { mkdir(w, "nm"); let cases = [ ("nm/same", 0o644, bytes(1, 300_000)), - ("nm/mode", 0o644, bytes(2, 5_000)), + ("nm/mode", 0o644, bytes(2, 200_000)), ("nm/byte", 0o644, bytes(3, 5_000)), ("nm/linked", 0o644, bytes(4, 5_000)), ("top", 0o755, bytes(5, 70_000)), @@ -283,10 +296,8 @@ fn a_relinked_file_with_saved_bytes_reuses_the_object() { mkdir(w, "elsewhere"); put(w, "elsewhere/copy", 0o644, &cases[4].2); let (second, work) = commit(w); - assert_eq!( - work.files_reused, 4, - "nm/same, nm/mode, top, elsewhere/copy" - ); + // top and elsewhere/copy are small: built again, the same object. + assert_eq!(work.files_reused, 2, "nm/same, nm/mode"); let after: Vec<(u64, InodeValue)> = cases .iter() .map(|(path, _, _)| committed(&s.store, w, path)) @@ -321,7 +332,7 @@ fn a_relinked_file_with_saved_bytes_reuses_the_object() { #[test] fn a_recorded_root_the_store_does_not_hold_is_not_reused() { let s = setup("file-digests-missing"); - let data = bytes(9, 40_000); + let data = bytes(9, CHUNKED + 40_000); let digest = *blake3::hash(&data).as_bytes(); s.store .history() diff --git a/engine/crates/layerfs-persistence/README.md b/engine/crates/layerfs-persistence/README.md index 378d006302..a0dc96b881 100644 --- a/engine/crates/layerfs-persistence/README.md +++ b/engine/crates/layerfs-persistence/README.md @@ -42,7 +42,8 @@ PostgreSQL unchanged except `hint_insert_binding` and `statistics_totals`. key order, each in one `REPEATABLE READ` snapshot after the owner check; writes take the lane like every write. A schema without the acquisition tables refuses every unit (`BackendUnavailable`). Both table sets are - schema version 1; the recorded schema source says which. + schema version 2 (#91); the recorded schema source says which. A schema + version 1 schema is refused at open. - **Transactions.** Reads are `REPEATABLE READ READ ONLY`; writes are `READ COMMITTED`, never SERIALIZABLE, with `SET LOCAL synchronous_commit` (default `off`) in the same round trip as `BEGIN`. Every write first locks the @@ -55,7 +56,7 @@ PostgreSQL unchanged except `hint_insert_binding` and `statistics_totals`. - **Objects.** Pack bodies are whole `bytea` values stored `EXTERNAL`; ranged reads use `substring`. A publication copies its pack bodies with `COPY BINARY` in a write that does not wait for the lane, then adds - locations, values, signatures and totals in the short lane transaction. A + locations, inode value blocks, signatures and totals in the short lane transaction. A failed second step leaves unreferenced packs, never reused ids. - **Removing a project.** `HistoryCatalog::remove_layerstack` deletes one stack's Layers, Branches, Commits and lineage, and their logical bytes from @@ -68,6 +69,20 @@ PostgreSQL unchanged except `hint_insert_binding` and `statistics_totals`. seal/snapshot/reclamation (no file), and the host SQLite cache #420 found necessary for one round trip per Commit. +Inode values and binding hints live in append-only blocks since schema +version 2 (#91; `inode_value_block`, `binding_hint_block`, see +[metadata and objects](../../../docs/architecture/engine/store/metadata-and-objects.md#inode-value-and-binding-hint-blocks)): +SQLite schema versions 36 to 38 (42 to 44 with the acquisition tables) and +PostgreSQL `SCHEMA_VERSION` 2, which also holds PostgreSQL object locations in +narrow rows under an 8-byte-prefix index (#90). A schema version 1 Store is +refused at open and never converted in place: SQLite 16 to 18 and 22 to 24 +with `Refused` naming the offline conversion, PostgreSQL 1 as `Malformed`. +Converting a SQLite Store is optional: `tools/store_blocks.py migrate + ` writes a schema version 2 copy (checked, +`integrity_check`, vacuumed) and leaves the source untouched; without it the +Store stays readable by a 2.0 release built before #91. A PostgreSQL schema +version 1 Store has no conversion tool. + The twelve logical tables are `pack`; `store_policy`, `object_location`, `inode_value`, `content_signature`; and `history_meta`, `scope_allocator`, `layer_stack`, `layer`, `commit`, `branch`, `workspace_stage`. diff --git a/engine/crates/layerfs-persistence/sql/postgresql/acquisition.sql b/engine/crates/layerfs-persistence/sql/postgresql/acquisition.sql index e3db833235..6ed5423616 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/acquisition.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/acquisition.sql @@ -2,10 +2,10 @@ CREATE TABLE init_operation ( operation_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, owner_epoch bigint NOT NULL CHECK (owner_epoch > 0), phase bigint NOT NULL CHECK (phase BETWEEN 1 AND 6), - source_device bytea NOT NULL CHECK (octet_length(source_device) = 8), - source_inode bytea NOT NULL CHECK (octet_length(source_inode) = 8), - stack_id bytea NOT NULL CHECK (octet_length(stack_id) = 16), - scope bytea NOT NULL CHECK (octet_length(scope) = 32), + source_device bytea STORAGE PLAIN NOT NULL CHECK (octet_length(source_device) = 8), + source_inode bytea STORAGE PLAIN NOT NULL CHECK (octet_length(source_inode) = 8), + stack_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(stack_id) = 16), + scope bytea STORAGE PLAIN NOT NULL CHECK (octet_length(scope) = 32), held_rows bigint NOT NULL DEFAULT 0 CHECK (held_rows >= 0), held_bytes bigint NOT NULL DEFAULT 0 CHECK (held_bytes >= 0), peak_rows bigint NOT NULL DEFAULT 0 CHECK (peak_rows >= held_rows), @@ -16,26 +16,26 @@ CREATE TABLE init_operation ( CREATE TABLE init_entry ( operation_id bigint NOT NULL REFERENCES init_operation (operation_id), parent_position bigint NOT NULL CHECK (parent_position >= -1), - name bytea NOT NULL CHECK (octet_length(name) <= 255), + name bytea STORAGE PLAIN NOT NULL CHECK (octet_length(name) <= 255), position bigint CHECK (position >= 0), kind bigint NOT NULL CHECK (kind BETWEEN 1 AND 3), canonical_position bigint CHECK (canonical_position > 0), - metadata_root bytea NOT NULL CHECK (octet_length(metadata_root) = 32), - content_root bytea CHECK (octet_length(content_root) = 32), - host_path bytea CHECK (octet_length(host_path) BETWEEN 1 AND 4096), - host_identity bytea CHECK (octet_length(host_identity) = 60), + metadata_root bytea STORAGE PLAIN NOT NULL CHECK (octet_length(metadata_root) = 32), + content_root bytea STORAGE PLAIN CHECK (octet_length(content_root) = 32), + host_path bytea STORAGE PLAIN CHECK (octet_length(host_path) BETWEEN 1 AND 4096), + host_identity bytea STORAGE PLAIN CHECK (octet_length(host_identity) = 60), PRIMARY KEY (operation_id, parent_position, name) ); CREATE UNIQUE INDEX init_entry_directory ON init_entry (operation_id, position) WHERE kind = 2; CREATE TABLE init_host_file ( operation_id bigint NOT NULL REFERENCES init_operation (operation_id), canonical_position bigint NOT NULL CHECK (canonical_position > 0), - device bytea NOT NULL CHECK (octet_length(device) = 8), - inode bytea NOT NULL CHECK (octet_length(inode) = 8), - evidence bytea NOT NULL CHECK (octet_length(evidence) = 44), - host_path bytea NOT NULL CHECK (octet_length(host_path) BETWEEN 1 AND 4096), + device bytea STORAGE PLAIN NOT NULL CHECK (octet_length(device) = 8), + inode bytea STORAGE PLAIN NOT NULL CHECK (octet_length(inode) = 8), + evidence bytea STORAGE PLAIN NOT NULL CHECK (octet_length(evidence) = 44), + host_path bytea STORAGE PLAIN NOT NULL CHECK (octet_length(host_path) BETWEEN 1 AND 4096), aliases bigint NOT NULL DEFAULT 0 CHECK (aliases >= 0), - file_root bytea CHECK (octet_length(file_root) = 32), + file_root bytea STORAGE PLAIN CHECK (octet_length(file_root) = 32), PRIMARY KEY (operation_id, canonical_position) ); CREATE UNIQUE INDEX init_host_file_identity ON init_host_file (operation_id, device, inode); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/digest_roots.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/digest_roots.sql index 1bfa787a5e..06a2eca8f4 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/digest_roots.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/digest_roots.sql @@ -1 +1 @@ -SELECT k.i, f.root FROM unnest($1::bytea[], $2::bigint[]) WITH ORDINALITY AS k(digest, length, i) LEFT JOIN ({s}.file_digest f JOIN {s}.object_location o ON o.object_id = f.root) ON f.digest = k.digest AND f.length = k.length ORDER BY k.i; +SELECT k.i, r.root FROM unnest($1::bytea[], $2::bigint[]) WITH ORDINALITY AS k(digest, length, i) LEFT JOIN LATERAL (SELECT f.root FROM {s}.file_digest f WHERE f.digest = k.digest AND f.length = k.length AND EXISTS (SELECT 1 FROM {s}.object_location o WHERE ('x' || encode(substring(o.object_id FROM 1 FOR 8), 'hex'))::bit(64)::bigint = ('x' || encode(substring(f.root FROM 1 FOR 8), 'hex'))::bit(64)::bigint AND o.object_id = f.root OFFSET 0)) r ON true ORDER BY k.i; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_binding.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_binding.sql deleted file mode 100644 index cccdf440bf..0000000000 --- a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_binding.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO binding_hint(child, parent, name) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_bindings.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_bindings.sql deleted file mode 100644 index f9e7b100a0..0000000000 --- a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_bindings.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO binding_hint(child, parent, name) SELECT * FROM unnest($1::bigint[], $2::bigint[], $3::bytea[]) ON CONFLICT DO NOTHING; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_block.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_block.sql new file mode 100644 index 0000000000..fcee570cb3 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_block.sql @@ -0,0 +1 @@ +INSERT INTO binding_hint_block(grp, seq, body) SELECT $1::bigint, coalesce(max(seq), 0) + 1, $2::bytea FROM binding_hint_block WHERE grp = $1::bigint; diff --git a/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_blocks.sql b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_blocks.sql new file mode 100644 index 0000000000..56fb687f60 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/postgresql/queries/history/hint_insert_blocks.sql @@ -0,0 +1,4 @@ +INSERT INTO binding_hint_block(grp, seq, body) +SELECT n.g, coalesce((SELECT max(b.seq) FROM binding_hint_block b WHERE b.grp = n.g), 0) + + row_number() OVER (PARTITION BY n.g ORDER BY n.o), n.body +FROM unnest($1::bigint[], $2::bytea[]) WITH ORDINALITY n(g, body, o); diff --git a/engine/crates/layerfs-persistence/sql/postgresql/schema.sql b/engine/crates/layerfs-persistence/sql/postgresql/schema.sql index b94393940a..5b15c1bb69 100644 --- a/engine/crates/layerfs-persistence/sql/postgresql/schema.sql +++ b/engine/crates/layerfs-persistence/sql/postgresql/schema.sql @@ -4,7 +4,7 @@ CREATE TABLE store_lane ( INSERT INTO store_lane (id) VALUES (1); CREATE TABLE store_policy ( id bigint PRIMARY KEY CHECK (id = 1), - schema_version bigint NOT NULL CHECK (schema_version = 1), + schema_version bigint NOT NULL CHECK (schema_version = 2), format_profile bigint NOT NULL CHECK (format_profile = 1), small_file_threshold_bytes bigint NOT NULL CHECK (small_file_threshold_bytes BETWEEN 131072 AND 1048576), whole_file_delta_max_depth bigint NOT NULL CHECK (whole_file_delta_max_depth BETWEEN 0 AND 50), @@ -27,22 +27,26 @@ BEGIN RAISE EXCEPTION 'published pack immutable'; END $$; CREATE TRIGGER pack_immutable BEFORE UPDATE OR DELETE ON pack FOR EACH ROW EXECUTE FUNCTION pack_immutable(); CREATE TABLE object_location ( - object_id bytea PRIMARY KEY CHECK (octet_length(object_id) = 32), - role bigint NOT NULL CHECK (role BETWEEN 1 AND 13), - canonical_length bigint NOT NULL CHECK (canonical_length BETWEEN 1 AND 16777216), pack_id bigint NOT NULL REFERENCES pack (pack_id), - group_number bigint NOT NULL CHECK (group_number BETWEEN 0 AND 255), - record_number bigint NOT NULL CHECK (record_number BETWEEN 0 AND 8190) -); -CREATE TABLE inode_value ( - ordinal bigint PRIMARY KEY CHECK (ordinal BETWEEN 1 AND 4294967295), - value bytea NOT NULL CHECK (octet_length(value) = 73) + canonical_length integer NOT NULL CHECK (canonical_length BETWEEN 1 AND 16777216), + role smallint NOT NULL CHECK (role BETWEEN 1 AND 13), + group_number smallint NOT NULL CHECK (group_number BETWEEN 0 AND 255), + record_number smallint NOT NULL CHECK (record_number BETWEEN 0 AND 8190), + object_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(object_id) = 32) +); +CREATE INDEX object_location_prefix ON object_location + ((('x' || encode(substring(object_id FROM 1 FOR 8), 'hex'))::bit(64)::bigint)); +CREATE TABLE inode_value_block ( + first_ordinal bigint PRIMARY KEY CHECK (first_ordinal BETWEEN 1 AND 4294967295), + last_ordinal bigint NOT NULL + CHECK (last_ordinal BETWEEN first_ordinal AND 4294967295 AND last_ordinal - first_ordinal < 255), + body bytea STORAGE PLAIN NOT NULL CHECK (octet_length(body) BETWEEN 9 AND 4000) ); CREATE TABLE content_signature ( slot bigint PRIMARY KEY CHECK (slot BETWEEN 0 AND 8191), stamp bigint NOT NULL CHECK (stamp > 0 AND (stamp - 1) % 8192 = slot), - object_id bytea NOT NULL REFERENCES object_location (object_id), - signature bytea NOT NULL CHECK (octet_length(signature) = 32) + object_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(object_id) = 32), + signature bytea STORAGE PLAIN NOT NULL CHECK (octet_length(signature) = 32) ); CREATE TABLE store_totals ( id bigint PRIMARY KEY CHECK (id = 1), @@ -65,18 +69,18 @@ CREATE TABLE history_meta ( catalog_incarnation bigint NOT NULL CHECK (catalog_incarnation BETWEEN 1 AND 9223372036854775807), binding_key bytea NOT NULL CHECK (octet_length(binding_key) BETWEEN 1 AND 128), identity_format bigint NOT NULL CHECK (identity_format = 3), - schema_version bigint NOT NULL CHECK (schema_version = 1), + schema_version bigint NOT NULL CHECK (schema_version = 2), schema_source text NOT NULL, schema_definition text NOT NULL ); CREATE TABLE layer_stack ( - layer_stack_id bytea PRIMARY KEY + layer_stack_id bytea STORAGE PLAIN PRIMARY KEY CHECK (octet_length(layer_stack_id) = 17 AND get_byte(layer_stack_id, 0) = 49), - name text COLLATE "C" NOT NULL + name text STORAGE PLAIN COLLATE "C" NOT NULL CHECK (length(name) BETWEEN 1 AND 63 AND name ~ '^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$'), - scope_id bytea NOT NULL CHECK (octet_length(scope_id) = 32), - profile_id bytea NOT NULL CHECK (octet_length(profile_id) = 32), - head_layer_id bytea NOT NULL + scope_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(scope_id) = 32), + profile_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(profile_id) = 32), + head_layer_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(head_layer_id) = 33 AND get_byte(head_layer_id, 0) = 50) ); CREATE TABLE layer ( @@ -125,15 +129,15 @@ CREATE TABLE "commit" ( UNIQUE (layer_stack_id, commit_id) ); CREATE TABLE branch ( - branch_id bytea PRIMARY KEY CHECK (octet_length(branch_id) = 17 AND get_byte(branch_id, 0) = 17), - layer_stack_id bytea NOT NULL + branch_id bytea STORAGE PLAIN PRIMARY KEY CHECK (octet_length(branch_id) = 17 AND get_byte(branch_id, 0) = 17), + layer_stack_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(layer_stack_id) = 17 AND get_byte(layer_stack_id, 0) = 49), - name text COLLATE "C" NOT NULL + name text STORAGE PLAIN COLLATE "C" NOT NULL CHECK (length(name) BETWEEN 1 AND 63 AND name ~ '^[a-z0-9]([a-z0-9._-]*[a-z0-9])?$'), - base_layer_id bytea NOT NULL + base_layer_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(base_layer_id) = 33 AND get_byte(base_layer_id, 0) = 50), - forked_from_id bytea CHECK (forked_from_id IS NULL OR octet_length(forked_from_id) = 33), - head_commit_id bytea + forked_from_id bytea STORAGE PLAIN CHECK (forked_from_id IS NULL OR octet_length(forked_from_id) = 33), + head_commit_id bytea STORAGE PLAIN CHECK (head_commit_id IS NULL OR (octet_length(head_commit_id) = 33 AND get_byte(head_commit_id, 0) = 18)), UNIQUE (layer_stack_id, branch_id), UNIQUE (layer_stack_id, name) @@ -159,15 +163,15 @@ ALTER TABLE branch ADD FOREIGN KEY (layer_stack_id, base_layer_id) ALTER TABLE branch ADD FOREIGN KEY (layer_stack_id, head_commit_id) REFERENCES "commit" (layer_stack_id, commit_id) DEFERRABLE INITIALLY DEFERRED; CREATE TABLE scope_allocator ( - scope_id bytea PRIMARY KEY CHECK (octet_length(scope_id) = 32), + scope_id bytea STORAGE PLAIN PRIMARY KEY CHECK (octet_length(scope_id) = 32), highwater bigint NOT NULL CHECK (highwater BETWEEN 0 AND 9223372036854775807), - authority_id bytea NOT NULL CHECK (octet_length(authority_id) = 32) + authority_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(authority_id) = 32) ); -CREATE TABLE binding_hint ( - child bigint NOT NULL CHECK (child != 0), - parent bigint NOT NULL CHECK (parent != 0), - name bytea NOT NULL CHECK (octet_length(name) BETWEEN 1 AND 255), - PRIMARY KEY (child, parent, name) +CREATE TABLE binding_hint_block ( + grp bigint NOT NULL CHECK (grp >= 0), + seq bigint NOT NULL CHECK (seq >= 1), + body bytea STORAGE PLAIN NOT NULL CHECK (octet_length(body) BETWEEN 9 AND 4000), + PRIMARY KEY (grp, seq) ); CREATE UNIQUE INDEX layer_stack_names ON layer_stack (name); CREATE UNIQUE INDEX layers_genesis ON layer (layer_stack_id) WHERE parent_layer_id IS NULL; @@ -180,20 +184,20 @@ BEGIN UPDATE store_totals SET logical = logical + NEW.logical WHERE id = 1; RETU CREATE TRIGGER commit_logical AFTER INSERT ON "commit" FOR EACH ROW EXECUTE FUNCTION add_logical(); CREATE TRIGGER layer_logical AFTER INSERT ON layer FOR EACH ROW EXECUTE FUNCTION add_logical(); CREATE TABLE layer_lineage ( - layer_id bytea PRIMARY KEY CHECK (octet_length(layer_id) = 33 AND get_byte(layer_id, 0) = 50), - layer_stack_id bytea NOT NULL CHECK (octet_length(layer_stack_id) = 17), + layer_id bytea STORAGE PLAIN PRIMARY KEY CHECK (octet_length(layer_id) = 33 AND get_byte(layer_id, 0) = 50), + layer_stack_id bytea STORAGE PLAIN NOT NULL CHECK (octet_length(layer_stack_id) = 17), height bigint NOT NULL CHECK (height >= 0), overwrote_count bigint NOT NULL CHECK (overwrote_count >= 0) ); CREATE TABLE commit_lineage ( - commit_id bytea PRIMARY KEY CHECK (octet_length(commit_id) = 33 AND get_byte(commit_id, 0) = 18), - run bytea NOT NULL CHECK (octet_length(run) = 33), + commit_id bytea STORAGE PLAIN PRIMARY KEY CHECK (octet_length(commit_id) = 33 AND get_byte(commit_id, 0) = 18), + run bytea STORAGE PLAIN NOT NULL CHECK (octet_length(run) = 33), depth bigint NOT NULL CHECK (depth >= 1), newest bigint NOT NULL CHECK (newest >= 0), - merged bytea CHECK (merged IS NULL OR octet_length(merged) = 33) + merged bytea STORAGE PLAIN CHECK (merged IS NULL OR octet_length(merged) = 33) ); CREATE TABLE absorbed ( - run bytea NOT NULL CHECK (octet_length(run) = 33), + run bytea STORAGE PLAIN NOT NULL CHECK (octet_length(run) = 33), depth bigint NOT NULL CHECK (depth >= 1), height bigint NOT NULL CHECK (height >= 0), PRIMARY KEY (run, depth) @@ -209,8 +213,8 @@ CREATE UNIQUE INDEX commit_runs ON commit_lineage (run, depth); CREATE INDEX run_merges ON commit_lineage (run, depth) WHERE merged IS NOT NULL; CREATE INDEX absorbed_heights ON absorbed (run, height); CREATE TABLE file_digest ( - digest bytea NOT NULL CHECK (octet_length(digest) = 32), + digest bytea STORAGE PLAIN NOT NULL CHECK (octet_length(digest) = 32), length bigint NOT NULL CHECK (length >= 0), - root bytea NOT NULL CHECK (octet_length(root) = 32), + root bytea STORAGE PLAIN NOT NULL CHECK (octet_length(root) = 32), PRIMARY KEY (digest, length) ); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/history.sql b/engine/crates/layerfs-persistence/sql/sqlite/history.sql index 9de411407b..6a587b78cc 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/history.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/history.sql @@ -141,12 +141,12 @@ CREATE TABLE scope_allocator ( authority_id BLOB NOT NULL CHECK (length(authority_id) = 32) ) STRICT, WITHOUT ROWID; -CREATE TABLE binding_hint ( - child INTEGER NOT NULL CHECK (child != 0), - parent INTEGER NOT NULL CHECK (parent != 0), - name BLOB NOT NULL CHECK (length(name) BETWEEN 1 AND 255), - PRIMARY KEY (child, parent, name) -) STRICT, WITHOUT ROWID; +CREATE TABLE binding_hint_block ( + grp INTEGER NOT NULL CHECK (grp >= 0), + seq INTEGER NOT NULL CHECK (seq >= 1), + body BLOB NOT NULL CHECK (length(body) BETWEEN 9 AND 4000), + PRIMARY KEY (grp, seq) +) STRICT; CREATE UNIQUE INDEX layer_stack_names ON layer_stack(name); CREATE UNIQUE INDEX layer_identity ON layer(layer_stack_id, layer_id); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/metadata.sql b/engine/crates/layerfs-persistence/sql/sqlite/metadata.sql index 0638287583..64671b6172 100644 --- a/engine/crates/layerfs-persistence/sql/sqlite/metadata.sql +++ b/engine/crates/layerfs-persistence/sql/sqlite/metadata.sql @@ -16,9 +16,11 @@ CREATE TABLE object_location ( group_number INTEGER NOT NULL CHECK(group_number BETWEEN 0 AND 255), record_number INTEGER NOT NULL CHECK(record_number BETWEEN 0 AND 8190) ) STRICT, WITHOUT ROWID; -CREATE TABLE inode_value ( - ordinal INTEGER PRIMARY KEY CHECK(ordinal BETWEEN 1 AND 4294967295), - value BLOB NOT NULL CHECK(length(value)=73) +CREATE TABLE inode_value_block ( + first_ordinal INTEGER PRIMARY KEY CHECK(first_ordinal BETWEEN 1 AND 4294967295), + last_ordinal INTEGER NOT NULL + CHECK(last_ordinal BETWEEN first_ordinal AND 4294967295 AND last_ordinal-first_ordinal<255), + body BLOB NOT NULL CHECK(length(body) BETWEEN 9 AND 4000) ) STRICT; CREATE TABLE content_signature ( slot INTEGER PRIMARY KEY CHECK(slot BETWEEN 0 AND 8191), diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_bindings_of.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_bindings_of.sql deleted file mode 100644 index cc14d08a3e..0000000000 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_bindings_of.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT child, parent, name FROM binding_hint WHERE child = $1; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_blocks_of.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_blocks_of.sql new file mode 100644 index 0000000000..6e56af00e6 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_blocks_of.sql @@ -0,0 +1 @@ +SELECT body FROM binding_hint_block WHERE grp = $1 ORDER BY seq; diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_insert_binding.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_insert_binding.sql deleted file mode 100644 index a5489a9af4..0000000000 --- a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_insert_binding.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT OR IGNORE INTO binding_hint(child, parent, name) VALUES ($1, $2, $3); diff --git a/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_insert_block.sql b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_insert_block.sql new file mode 100644 index 0000000000..e71d9381f0 --- /dev/null +++ b/engine/crates/layerfs-persistence/sql/sqlite/queries/history/hint_insert_block.sql @@ -0,0 +1 @@ +INSERT INTO binding_hint_block(grp, seq, body) SELECT $1, coalesce(max(seq), 0) + 1, $2 FROM binding_hint_block WHERE grp = $1; diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs index d955303c57..dd5b75edb9 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/history/statements.rs @@ -6,8 +6,8 @@ use crate::backend::records::BackendError; pub(crate) fn statement(key: &str) -> Result<&'static str, BackendError> { match key { - "hint_insert_binding" => Ok(include_str!( - "../../../../sql/postgresql/queries/history/hint_insert_binding.sql" + "hint_insert_block" => Ok(include_str!( + "../../../../sql/postgresql/queries/history/hint_insert_block.sql" )), "statistics_totals" => Ok(include_str!( "../../../../sql/postgresql/queries/history/statistics_totals.sql" @@ -30,8 +30,8 @@ pub(crate) fn qualified(key: &str) -> Option<&'static str> { /// The array form of a per-row statement, where one exists. pub(crate) fn arrays(key: &str) -> Option<&'static str> { match key { - "hint_insert_bindings" => Some(include_str!( - "../../../../sql/postgresql/queries/history/hint_insert_bindings.sql" + "hint_insert_blocks" => Some(include_str!( + "../../../../sql/postgresql/queries/history/hint_insert_blocks.sql" )), "digest_insert_rows" => Some(include_str!( "../../../../sql/postgresql/queries/history/digest_insert_rows.sql" diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs index d6f00bdf89..5ac3dd4b71 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs @@ -49,11 +49,22 @@ pub(crate) fn locate( .into_iter() .map(|id| id.as_bytes().to_vec()) .collect(); + let keys: Vec = ids + .iter() + .map(|id| ObjectId::from_bytes(id).map(|id| super::super::objects::publish::prefix(&id))) + .collect::>() + .map_err(|_| BackendError::Integrity)?; + // Each id finds its rows through the prefix index, then the full id is + // compared (#90); `OFFSET 0` keeps that a probe per id rather than a hash + // of the whole table. let sql = format!( - "SELECT o.object_id,o.role,o.canonical_length,o.group_number,o.record_number,p.pack_id,p.domain,p.digest,p.length FROM {s}.object_location o JOIN {s}.pack p ON p.pack_id=o.pack_id WHERE o.object_id = ANY($1::bytea[])", + "SELECT o.object_id,o.role::bigint,o.canonical_length::bigint,o.group_number::bigint,o.record_number::bigint,p.pack_id,p.domain,p.digest,p.length \ + FROM unnest($1::bytea[],$2::bigint[]) k(i,k) CROSS JOIN LATERAL (SELECT * FROM {s}.object_location x \ + WHERE ('x'||encode(substring(x.object_id FROM 1 FOR 8),'hex'))::bit(64)::bigint=k.k AND x.object_id=k.i OFFSET 0) o \ + JOIN {s}.pack p ON p.pack_id=o.pack_id", s = tx.schema ); - let rows = tx.typed(&sql, &[&ids], ids.len() as u64 * 32)?; + let rows = tx.typed(&sql, &[&ids, &keys], ids.len() as u64 * 40)?; for row in rows { let r = super::super::session::rows::record(&row)?; out.push(location(&r)?); @@ -82,27 +93,44 @@ fn location(r: &Record) -> Result { }) } -/// Every stored value among ascending distinct ordinals, in ordinal order. +/// Every stored value among ascending distinct ordinals, in ordinal order, +/// in one statement that names the schema. Each run of consecutive ordinals +/// seeks the last block starting at or before it and takes the blocks from +/// there to its end; each block is returned once, in ordinal order. pub(crate) fn inode_values( tx: &Transaction<'_>, ordinals: &[u32], out: &mut Vec, ) -> Result<(), PersistenceError> { - let ordinals: Vec = ordinals.iter().map(|o| i64::from(*o)).collect(); + let (mut lows, mut highs) = (Vec::new(), Vec::new()); + for &ordinal in ordinals { + match highs.last_mut() { + Some(high) if *high + 1 == i64::from(ordinal) => *high += 1, + _ => { + lows.push(i64::from(ordinal)); + highs.push(i64::from(ordinal)); + } + } + } let sql = format!( - "SELECT ordinal,value FROM {}.inode_value WHERE ordinal = ANY($1::bigint[]) ORDER BY ordinal", - tx.schema + "SELECT b.first_ordinal,b.last_ordinal,b.body FROM {s}.inode_value_block b WHERE b.first_ordinal IN (\ + SELECT x.first_ordinal FROM unnest($1::bigint[],$2::bigint[]) r(lo,hi) JOIN {s}.inode_value_block x \ + ON x.first_ordinal BETWEEN coalesce((SELECT max(y.first_ordinal) FROM {s}.inode_value_block y \ + WHERE y.first_ordinal<=r.lo),0) AND r.hi) ORDER BY b.first_ordinal", + s = tx.schema ); - let rows = tx.typed(&sql, &[&ordinals], ordinals.len() as u64 * 8)?; - for row in rows { - let r = super::super::session::rows::record(&row)?; - let ordinal = u32::try_from(r.get::(0)?).map_err(|_| BackendError::Integrity)?; - let value = r - .get::>(1)? - .try_into() - .map_err(|_| BackendError::Integrity)?; - out.push(InodeValueRow { ordinal, value }); - } + let rows = tx.typed(&sql, &[&lows, &highs], lows.len() as u64 * 16)?; + let ordinal = |v: i64| u32::try_from(v).map_err(|_| BackendError::Integrity); + crate::blocks::values::pick( + ordinals, + rows.iter().map(|row| { + let first: i64 = row.try_get(0).map_err(|_| BackendError::Integrity)?; + let last: i64 = row.try_get(1).map_err(|_| BackendError::Integrity)?; + let body: Vec = row.try_get(2).map_err(|_| BackendError::Integrity)?; + Ok((ordinal(first)?, ordinal(last)?, body)) + }), + out, + )?; Ok(()) } diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/objects/publish.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/objects/publish.rs index 3174380c3b..e5b149360d 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/objects/publish.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/objects/publish.rs @@ -2,9 +2,14 @@ //! first, by `COPY BINARY`, before the lane row is taken: their ids were //! reserved, so no other writer can touch those rows, and a slow body upload //! never holds the lane. The lane row is then locked and one data-modifying -//! statement adds the locations, inode values, signatures and totals; the +//! statement adds the locations, inode value blocks (#91), signatures and totals; the //! COMMIT acknowledges packs and rows together. A refused or failed step //! rolls the whole publication back; pack ids are never reused. +//! +//! A location is inserted when no row of its id exists (#90): the table has +//! no unique key on the id, only a B-tree on its first eight bytes, and +//! uniqueness holds because every location insert runs under the lane row. +//! A signature must name a location of this publication or a stored one. use super::super::session::transaction::Transaction; use crate::backend::records::BackendError; use layerfs_content::ObjectId; @@ -43,26 +48,48 @@ pub(crate) fn run(tx: &Transaction<'_>, batch: &Publication) -> Resultcardinality($9::bigint[]) \ +UNION ALL SELECT NULL::bytea WHERE EXISTS (SELECT 1 FROM unnest($16::bytea[],$20::bigint[]) s(i,k) \ +WHERE NOT EXISTS (SELECT 1 FROM unnest($1::bytea[]) b(i) WHERE b.i=s.i) \ +AND NOT EXISTS (SELECT 1 FROM object_location x WHERE ('x'||encode(substring(x.object_id FROM 1 FOR 8),'hex'))::bit(64)::bigint=s.k AND x.object_id=s.i OFFSET 0))"; + +/// The index key of an object id: its first eight bytes as a signed +/// big-endian `bigint`, as the schema's `object_location_prefix` computes it. +pub(crate) fn prefix(id: &ObjectId) -> i64 { + let mut head = [0_u8; 8]; + head.copy_from_slice(&id.as_bytes()[..8]); + i64::from_be_bytes(head) +} pub(crate) fn rows(tx: &Transaction<'_>, batch: &Publication) -> Result { + // Ids are distinct: `storage::publication` refuses a repeated one. let objects = &batch.objects; + let keys: Vec = objects.iter().map(|o| prefix(&o.object_id)).collect(); let ids: Vec<&[u8]> = objects .iter() .map(|o| o.object_id.as_bytes().as_slice()) @@ -86,9 +113,11 @@ pub(crate) fn rows(tx: &Transaction<'_>, batch: &Publication) -> Result(); - let values = &batch.inode_values; - let ordinals: Vec = values.iter().map(|r| i64::from(r.ordinal)).collect(); - let bodies: Vec<&[u8]> = values.iter().map(|r| r.value.as_slice()).collect(); + // Blocks are encoded here, before the statement that holds the lane. + let values = crate::blocks::values::encode(&batch.inode_values)?; + let ordinals: Vec = values.iter().map(|b| i64::from(b.first)).collect(); + let lasts: Vec = values.iter().map(|b| i64::from(b.last)).collect(); + let bodies: Vec<&[u8]> = values.iter().map(|b| b.body.as_slice()).collect(); let (release, first, count) = match batch.release_ordinals { Some((first, count)) => (true, i64::from(first), count as i64), None => (false, 0, 0), @@ -101,18 +130,42 @@ pub(crate) fn rows(tx: &Transaction<'_>, batch: &Publication) -> Result = signatures.iter().map(|r| r.signature.as_slice()).collect(); - let bytes = - objects.len() as u64 * 72 + values.len() as u64 * 81 + signatures.len() as u64 * 80 + 48; + let signed_keys: Vec = signatures.iter().map(|r| prefix(&r.object_id)).collect(); + let bytes = objects.len() as u64 * 72 + + values.iter().map(|b| b.body.len() as u64 + 16).sum::() + + signatures.len() as u64 * 80 + + 48; let mut inserted = BTreeSet::new(); for row in tx.typed( ROWS, &[ - &ids, &roles, &lengths, &packs, &groups, &records, &stored, &metadata, &ordinals, - &bodies, &first, &count, &release, &slots, &stamps, &signed, &digests, + &ids, + &roles, + &lengths, + &packs, + &groups, + &records, + &stored, + &metadata, + &ordinals, + &bodies, + &first, + &count, + &release, + &slots, + &stamps, + &signed, + &digests, + &lasts, + &keys, + &signed_keys, ], bytes, )? { - let id: Vec = row.try_get(0).map_err(|_| BackendError::Integrity)?; + // A NULL id: a block's ordinals were already stored, or a signature + // names no location; nothing commits. + let id: Option> = row.try_get(0).map_err(|_| BackendError::Integrity)?; + let id = id.ok_or(BackendError::Integrity)?; inserted.insert(ObjectId::from_bytes(&id).map_err(|_| BackendError::Integrity)?); } Ok(Published { diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs index faf2ac8923..2131a00d88 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs @@ -25,18 +25,21 @@ const ACQUISITION: &str = include_str!("../../../sql/postgresql/acquisition.sql" /// The acquisition tables, in name order. const ACQUISITION_TABLES: [&str; 3] = ["init_entry", "init_host_file", "init_operation"]; /// The engine-scoped schema version in `store_policy` and `history_meta`. -pub(crate) const SCHEMA_VERSION: i64 = 1; +/// Version 2 keeps inode values and binding hints in append-only blocks +/// (#91) and object locations in narrow rows under an 8-byte-prefix index +/// (#90); a version 1 schema is refused at open, never converted. +pub(crate) const SCHEMA_VERSION: i64 = 2; /// Every table of one Store schema. pub(crate) const TABLES: [&str; 19] = [ "absorbed", - "binding_hint", + "binding_hint_block", "branch", "commit", "commit_lineage", "content_signature", "file_digest", "history_meta", - "inode_value", + "inode_value_block", "layer", "layer_lineage", "layer_stack", diff --git a/engine/crates/layerfs-persistence/src/backend/records.rs b/engine/crates/layerfs-persistence/src/backend/records.rs index 23b234bcbe..efd1c9a57c 100644 --- a/engine/crates/layerfs-persistence/src/backend/records.rs +++ b/engine/crates/layerfs-persistence/src/backend/records.rs @@ -120,14 +120,24 @@ pub(crate) enum BackendError { ReadOnly, Capacity, Integrity, + /// A SQLite Store of schema version 1 (before #91): refused at open with + /// the offline conversion named, never misread. + SchemaVersion1, Unknown, } +/// The refusal a schema version 1 SQLite Store gets at open. +pub(crate) const SCHEMA_VERSION_1: &str = + "schema version 1 Store (one row per inode value and binding hint, before #91): \ +convert a closed copy offline with tools/store_blocks.py migrate"; impl From for layerfs_storage::port::PersistenceError { fn from(e: BackendError) -> Self { match e { BackendError::Unknown => Self::Uncertain, BackendError::Busy => Self::Busy, BackendError::Integrity => Self::Malformed, + BackendError::SchemaVersion1 => Self::Refused { + status: SCHEMA_VERSION_1.to_owned(), + }, _ => Self::Refused { status: format!("{e:?}"), }, @@ -145,6 +155,7 @@ impl From for layerfs_history::HistoryError { BackendError::ReadOnly => Self::ContinuityUnavailable, BackendError::Capacity => Self::Capacity("persistence"), BackendError::Integrity => Self::Integrity("persistence"), + BackendError::SchemaVersion1 => Self::Integrity(SCHEMA_VERSION_1), BackendError::Unknown => Self::UnknownOutcome, } } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs index 859950f603..15ed660ed5 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/metadata/values.rs @@ -1,18 +1,14 @@ -//! Stored inode values: one row an ordinal, read and written in bounded sets. +//! Stored inode values in append-only blocks (#91), read and written in +//! bounded sets. use crate::backend::{ records::{BackendError, Param}, Transaction, }; +use crate::blocks::values::{encode, pick}; use layerfs_storage::{location::InodeValueRow, port::*}; -/// Every stored value among ascending distinct ordinals, in ordinal order, in -/// one statement execution. The ordinals travel as one JSON parameter of -/// inclusive runs, so a leaf whose values were written together costs one -/// range scan and a scattered one a rowid seek for each run. -pub(crate) fn read( - tx: &Transaction<'_>, - ordinals: &[u32], - out: &mut Vec, -) -> Result<(), PersistenceError> { + +/// Ascending distinct ordinals as one JSON array of inclusive runs. +fn runs(ordinals: &[u32]) -> String { let mut runs = String::from("["); let mut index = 0; while index < ordinals.len() { @@ -29,48 +25,81 @@ pub(crate) fn read( runs.push_str(&format!("[{first},{last}]")); } runs.push(']'); + runs +} + +/// Every stored value among ascending distinct ordinals, in ordinal order, +/// in one statement execution. The ordinals travel as one JSON parameter of +/// inclusive runs; each run seeks the last block starting at or before it +/// and reads the blocks from there to its end, so a leaf whose values were +/// written together costs one or two blocks. +pub(crate) fn read( + tx: &Transaction<'_>, + ordinals: &[u32], + out: &mut Vec, +) -> Result<(), PersistenceError> { let rows = tx.query( - "SELECT v.ordinal,v.value FROM json_each(?1) r JOIN inode_value v ON v.ordinal BETWEEN json_extract(r.value,'$[0]') AND json_extract(r.value,'$[1]')", - vec![Param::Text(runs)], + "SELECT b.first_ordinal,b.last_ordinal,b.body FROM inode_value_block b WHERE b.first_ordinal IN (\ + SELECT x.first_ordinal FROM json_each(?1) r JOIN inode_value_block x \ + ON x.first_ordinal BETWEEN coalesce((SELECT max(y.first_ordinal) FROM inode_value_block y \ + WHERE y.first_ordinal<=json_extract(r.value,'$[0]')),0) AND json_extract(r.value,'$[1]')) \ + ORDER BY b.first_ordinal", + vec![Param::Text(runs(ordinals))], + )?; + let ordinal = |v: i64| u32::try_from(v).map_err(|_| BackendError::Integrity); + pick( + ordinals, + rows.into_iter().map(|mut row| { + Ok(( + ordinal(row.get::(0)?)?, + ordinal(row.get::(1)?)?, + row.take_bytes(2)?, + )) + }), + out, )?; - for row in rows { - let ordinal = u32::try_from(row.get::(0)?).map_err(|_| BackendError::Integrity)?; - let value = row - .get::>(1)? - .try_into() - .map_err(|_| BackendError::Integrity)?; - out.push(InodeValueRow { ordinal, value }); - } Ok(()) } -/// Inserts new rows in bounded multi-row statements. An ordinal that already -/// has a value fails the transaction: a stored value is never replaced. + +/// Inserts the values as new blocks in bounded multi-row statements. A +/// block whose ordinals a stored block already holds fails the +/// transaction: a stored value is never replaced. pub(crate) fn write(tx: &Transaction<'_>, rows: &[InodeValueRow]) -> Result<(), BackendError> { + let blocks = encode(rows)?; let limit = tx - .input_limit(2, 64, 8)? + .input_limit(3, 512, 16)? .min(layerfs_storage::policy::BATCH_OBJECT_LIMIT); - let mut remaining = rows; + let mut remaining = blocks.as_slice(); while !remaining.is_empty() { // Stable powers of two reuse prepared statements without padding rows. let count = 1usize << remaining.len().min(limit).ilog2(); let (page, tail) = remaining.split_at(count); remaining = tail; + // The last stored block starting at or before a new block's last + // ordinal is the only stored block that can overlap it. let sql = format!( - "INSERT INTO inode_value(ordinal,value) VALUES {}", - std::iter::repeat_n("(?,?)", page.len()) + "INSERT INTO inode_value_block(first_ordinal,last_ordinal,body) \ + SELECT column1,column2,column3 FROM (VALUES {}) n WHERE NOT EXISTS (\ + SELECT 1 FROM inode_value_block b WHERE b.first_ordinal=(SELECT max(c.first_ordinal) \ + FROM inode_value_block c WHERE c.first_ordinal<=n.column2) AND b.last_ordinal>=n.column1) \ + RETURNING first_ordinal", + std::iter::repeat_n("(?,?,?)", page.len()) .collect::>() .join(",") ); let params = page .iter() - .flat_map(|row| { + .flat_map(|block| { [ - Param::I64(i64::from(row.ordinal)), - Param::Bytes(row.value.to_vec()), + Param::I64(i64::from(block.first)), + Param::I64(i64::from(block.last)), + Param::Bytes(block.body.clone()), ] }) .collect(); - tx.query(&sql, params)?; + if tx.query(&sql, params)?.len() != page.len() { + return Err(BackendError::Integrity); + } } Ok(()) } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs index 0d25fc86a7..2ad0b355ea 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs @@ -85,14 +85,14 @@ pub(crate) fn check( let actual: Vec = rows.iter().map(|r| r.get(0)).collect::>()?; let mut expected = vec![ "absorbed", - "binding_hint", + "binding_hint_block", "branch", "commit", "commit_lineage", "content_signature", "file_digest", "history_meta", - "inode_value", + "inode_value_block", "layer", "layer_lineage", "layer_stack", @@ -144,7 +144,7 @@ impl super::session::connection::Session { })?; Ok(()) } - /// Adds the `binding_hint` table (#352 `49499abac`) to a Store written + /// Adds the binding hint table (#352 `49499abac`, blocks since #91) to a Store written /// before it existed, then records the current schema: an additive /// migration (REBUILD.md kind 2). Only a Store whose recorded schema is /// exactly the earlier one, and whose live schema still matches what it @@ -154,7 +154,7 @@ impl super::session::connection::Session { pub(crate) fn add_binding_hints(&self) -> Result<(), BackendError> { self.run::<_, BackendError>(true, |tx| { let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='binding_hint'", + "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='binding_hint_block'", vec![], )?; if !present.is_empty() { @@ -174,7 +174,7 @@ impl super::session::connection::Session { .replacen(SCRATCH, "", 1) .replacen(LINEAGE, "", 1); let table = HISTORY - .find("CREATE TABLE binding_hint (") + .find("CREATE TABLE binding_hint_block (") .and_then(|start| Some((start, start + HISTORY[start..].find(";\n")? + 2))) .map(|(start, end)| &HISTORY[start..end]) .ok_or(BackendError::Integrity)?; @@ -307,7 +307,7 @@ impl super::session::connection::Session { &self, catalog_id: layerfs_history::CatalogId, binding: &[u8], - ) -> Result { + ) -> Result<(u64, u64), BackendError> { self.run::<_,BackendError>(false,|tx| { let selection=(self.layout,self.acquisition); check(tx.connection,tx.work,selection)?; @@ -315,7 +315,9 @@ impl super::session::connection::Session { let r=rows.first().filter(|_|rows.len()==1).ok_or(BackendError::Integrity)?; let incarnation:i64=r.get(1)?; if r.get::>(0)?!=catalog_id.as_slice() || r.get::>(2)?!=binding || incarnation<=0 || r.get::(3)?!=3 || r.get::(4)?!=self.schema_version() || r.get::(5)?!=source(selection) || r.get::(6)?!=definition(tx.connection,tx.work)? {return Err(BackendError::Integrity);} - Ok(incarnation as u64) + let floor=tx.query("SELECT small_file_threshold_bytes FROM store_policy WHERE id=1",vec![])?; + let floor=u64::try_from(floor.first().ok_or(BackendError::Integrity)?.get::(0)?).map_err(|_|BackendError::Integrity)?; + Ok((incarnation as u64, floor)) }) } } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs index 2cecfb04e3..a7f0b20731 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs @@ -195,15 +195,19 @@ impl Session { .get::(0)?; // Versions 10 to 12 used the retired host acquisition schema and are refused. // Versions 1 to 6 kept inode values in pooled packs behind an - // ordinal catalogue. Such a Store is refused here, never misread. + // ordinal catalogue. Versions 16 to 18 and 22 to 24 (schema + // version 1) kept one row per inode value and binding hint; + // schema version 2 (#91) keeps them in blocks. Such a Store is + // refused here, never misread or converted. let acquisition = match stored { - 16..=18 => crate::SqliteAcquisitionSchema::Absent, - 22..=24 => crate::SqliteAcquisitionSchema::Tables, + 36..=38 => crate::SqliteAcquisitionSchema::Absent, + 42..=44 => crate::SqliteAcquisitionSchema::Tables, + 16..=18 | 22..=24 => return Err(BackendError::SchemaVersion1), _ => return Err(BackendError::Integrity), }; let layout = match stored - acquisition.version_offset() { - 16 => crate::SqlitePackLayout::Monolithic, - 17 => crate::SqlitePackLayout::GroupRows, + 36 => crate::SqlitePackLayout::Monolithic, + 37 => crate::SqlitePackLayout::GroupRows, _ => crate::SqlitePackLayout::GroupRowsIndexed, }; (layout, acquisition) diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs index 621e9ca652..e0b9edc4b6 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/query.rs @@ -21,11 +21,11 @@ pub(crate) fn history(key: &str) -> Result<&'static str, BackendError> { "allocation_scope_by_id" => Ok(include_str!( "../../../../sql/sqlite/queries/history/allocation_scope_by_id.sql" )), - "hint_insert_binding" => Ok(include_str!( - "../../../../sql/sqlite/queries/history/hint_insert_binding.sql" + "hint_insert_block" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/hint_insert_block.sql" )), - "hint_bindings_of" => Ok(include_str!( - "../../../../sql/sqlite/queries/history/hint_bindings_of.sql" + "hint_blocks_of" => Ok(include_str!( + "../../../../sql/sqlite/queries/history/hint_blocks_of.sql" )), "digest_insert" => Ok(include_str!( "../../../../sql/sqlite/queries/history/digest_insert.sql" diff --git a/engine/crates/layerfs-persistence/src/blocks/frame.rs b/engine/crates/layerfs-persistence/src/blocks/frame.rs new file mode 100644 index 0000000000..81e0025deb --- /dev/null +++ b/engine/crates/layerfs-persistence/src/blocks/frame.rs @@ -0,0 +1,135 @@ +//! Shared block framing: LEB128 varints, zig-zag integers and the CRC32C +//! (Castagnoli) trailer that seals each block body. +use crate::backend::records::BackendError; + +/// Largest body of one block: one row per 4 KiB SQLite page with no +/// overflow, two rows per 8 KiB PostgreSQL page, stored `PLAIN`. +pub(crate) const BODY_LIMIT: usize = 4_000; + +/// Width of the CRC32C trailer. +pub(crate) const CRC_BYTES: usize = 4; + +const fn table() -> [u32; 256] { + let mut table = [0u32; 256]; + let mut byte = 0; + while byte < 256 { + let mut crc = byte as u32; + let mut bit = 0; + while bit < 8 { + crc = if crc & 1 == 1 { + (crc >> 1) ^ 0x82F6_3B78 + } else { + crc >> 1 + }; + bit += 1; + } + table[byte] = crc; + byte += 1; + } + table +} +static TABLE: [u32; 256] = table(); + +/// CRC32C of `bytes` (reflected, initial and final XOR all ones). +pub(crate) fn crc32c(bytes: &[u8]) -> u32 { + let mut crc = u32::MAX; + for byte in bytes { + crc = TABLE[((crc ^ u32::from(*byte)) & 0xff) as usize] ^ (crc >> 8); + } + !crc +} + +/// Appends the trailer of a finished body. +pub(crate) fn seal(body: &mut Vec) { + let crc = crc32c(body); + body.extend_from_slice(&crc.to_le_bytes()); +} + +/// The bytes before a body's trailer, after checking it. +pub(crate) fn open(body: &[u8]) -> Result<&[u8], BackendError> { + let split = body + .len() + .checked_sub(CRC_BYTES) + .ok_or(BackendError::Integrity)?; + let (content, trailer) = body.split_at(split); + let stored = u32::from_le_bytes(trailer.try_into().map_err(|_| BackendError::Integrity)?); + if crc32c(content) != stored { + return Err(BackendError::Integrity); + } + Ok(content) +} + +/// LEB128 width of `value`. +pub(crate) const fn varint_len(mut value: u64) -> usize { + let mut len = 1; + while value >= 0x80 { + value >>= 7; + len += 1; + } + len +} + +pub(crate) fn put_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +pub(crate) const fn zigzag(value: i64) -> u64 { + ((value << 1) ^ (value >> 63)) as u64 +} + +pub(crate) const fn unzigzag(value: u64) -> i64 { + ((value >> 1) as i64) ^ -((value & 1) as i64) +} + +/// A strict reader over one opened body: every read is bounds checked, a +/// varint must be minimal and fit 64 bits, and the body must end exactly +/// where its last entry does. +pub(crate) struct Reader<'a> { + bytes: &'a [u8], + at: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(bytes: &'a [u8]) -> Self { + Self { bytes, at: 0 } + } + pub(crate) fn take(&mut self, n: usize) -> Result<&'a [u8], BackendError> { + let end = self.at.checked_add(n).ok_or(BackendError::Integrity)?; + let taken = self + .bytes + .get(self.at..end) + .ok_or(BackendError::Integrity)?; + self.at = end; + Ok(taken) + } + pub(crate) fn byte(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + pub(crate) fn varint(&mut self) -> Result { + let mut value = 0u64; + for index in 0..10 { + let byte = self.byte()?; + let bits = u64::from(byte & 0x7f); + // The tenth byte carries the top bit only; a zero last byte + // after the first is a non-minimal encoding. + if (index == 9 && byte > 1) || (index > 0 && byte == 0) { + return Err(BackendError::Integrity); + } + value |= bits << (7 * index); + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err(BackendError::Integrity) + } + pub(crate) fn finished(&self) -> Result<(), BackendError> { + match self.at == self.bytes.len() { + true => Ok(()), + false => Err(BackendError::Integrity), + } + } +} diff --git a/engine/crates/layerfs-persistence/src/blocks/hints.rs b/engine/crates/layerfs-persistence/src/blocks/hints.rs new file mode 100644 index 0000000000..790b7367a9 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/blocks/hints.rs @@ -0,0 +1,171 @@ +//! Binding hint blocks: the hints of one write for the 64 inode serials of +//! one group (`child >> 6`), appended as a new row `(group, seq)`. Rows are +//! never changed; a reader merges a group's rows and drops repeats. +//! +//! Body, format 1: the format byte `1`, a LEB128 entry count, the entries +//! in ascending `(child, parent, name)` order without repeats, then the +//! CRC32C of every byte before it. An entry is the child's LEB128 +//! difference from the previous entry's (from `group << 6` for the first), +//! the parent's zig-zag LEB128 difference from the previous entry's (from 0), +//! the length of the name prefix it shares with the previous entry's name +//! (one byte), and the LEB128 length and bytes of the rest of the name. +use super::frame::{self, Reader, BODY_LIMIT, CRC_BYTES}; +use crate::backend::records::BackendError; +use layerfs_history::records::BindingHint; + +const FORMAT: u8 = 1; +/// Inode serials per group. +pub(crate) const GROUP_BITS: u32 = 6; + +/// One encoded block of one group. +pub(crate) struct Block { + pub(crate) group: i64, + pub(crate) body: Vec, +} + +/// The group of `child`. +pub(crate) fn group_of(child: u64) -> i64 { + (child >> GROUP_BITS) as i64 +} + +fn valid(hint: &BindingHint) -> bool { + hint.child != 0 && hint.parent != 0 && (1..=255).contains(&hint.name.len()) +} + +fn shared(a: &[u8], b: &[u8]) -> usize { + a.iter().zip(b).take_while(|(x, y)| x == y).count() +} + +struct Open { + group: i64, + count: u64, + entries: Vec, + previous: Option<(u64, u64, Vec)>, +} + +impl Open { + fn new(group: i64) -> Self { + Self { + group, + count: 0, + entries: Vec::new(), + previous: None, + } + } + fn size(&self, extra: usize, count: u64) -> usize { + 1 + frame::varint_len(count) + self.entries.len() + extra + CRC_BYTES + } + fn push(&mut self, hint: &BindingHint) -> bool { + let (child, parent) = match &self.previous { + Some((child, parent, _)) => (*child, *parent), + None => ((self.group as u64) << GROUP_BITS, 0), + }; + let name: &[u8] = self.previous.as_ref().map_or(&[], |(_, _, name)| name); + let common = shared(name, &hint.name).min(255); + let mut entry = Vec::with_capacity(hint.name.len() + 16); + frame::put_varint(&mut entry, hint.child - child); + frame::put_varint( + &mut entry, + frame::zigzag((hint.parent as i64).wrapping_sub(parent as i64)), + ); + entry.push(common as u8); + frame::put_varint(&mut entry, (hint.name.len() - common) as u64); + entry.extend_from_slice(&hint.name[common..]); + if self.count > 0 && self.size(entry.len(), self.count + 1) > BODY_LIMIT { + return false; + } + self.entries.extend_from_slice(&entry); + self.count += 1; + self.previous = Some((hint.child, hint.parent, hint.name.clone())); + true + } + fn finish(self) -> Block { + let mut body = Vec::with_capacity(self.size(0, self.count)); + body.push(FORMAT); + frame::put_varint(&mut body, self.count); + body.extend_from_slice(&self.entries); + frame::seal(&mut body); + Block { + group: self.group, + body, + } + } +} + +/// Packs one write's hints into blocks, by group, in ascending order and +/// without repeats; a full block continues in another of the same group. +pub(crate) fn encode(rows: &[BindingHint]) -> Result, BackendError> { + let mut sorted: Vec<&BindingHint> = rows.iter().collect(); + if !sorted.iter().all(|hint| valid(hint)) { + return Err(BackendError::Integrity); + } + sorted.sort_unstable(); + sorted.dedup(); + let mut blocks = Vec::new(); + let mut open: Option = None; + for hint in sorted { + let group = group_of(hint.child); + if let Some(block) = open.as_mut() { + if block.group == group && block.push(hint) { + continue; + } + } + if let Some(block) = open.take() { + blocks.push(block.finish()); + } + let mut block = Open::new(group); + if !block.push(hint) { + return Err(BackendError::Integrity); + } + open = Some(block); + } + if let Some(block) = open { + blocks.push(block.finish()); + } + Ok(blocks) +} + +/// Every hint of one block of `group`, in its stored order. +pub(crate) fn decode(group: i64, body: &[u8]) -> Result, BackendError> { + if body.len() > BODY_LIMIT || group < 0 { + return Err(BackendError::Integrity); + } + let mut reader = Reader::new(frame::open(body)?); + if reader.byte()? != FORMAT { + return Err(BackendError::Integrity); + } + let count = reader.varint()?; + if count == 0 || count > BODY_LIMIT as u64 { + return Err(BackendError::Integrity); + } + let mut hints: Vec = Vec::with_capacity(count as usize); + let (mut child, mut parent) = ((group as u64) << GROUP_BITS, 0u64); + for _ in 0..count { + child = child + .checked_add(reader.varint()?) + .ok_or(BackendError::Integrity)?; + parent = (parent as i64).wrapping_add(frame::unzigzag(reader.varint()?)) as u64; + let common = usize::from(reader.byte()?); + let rest = usize::try_from(reader.varint()?).map_err(|_| BackendError::Integrity)?; + let previous: &[u8] = hints.last().map_or(&[], |hint| &hint.name); + if common > previous.len() { + return Err(BackendError::Integrity); + } + let mut name = previous[..common].to_vec(); + name.extend_from_slice(reader.take(rest)?); + let hint = BindingHint { + child, + parent, + name, + }; + if !valid(&hint) + || group_of(child) != group + || hints.last().is_some_and(|last| *last >= hint) + { + return Err(BackendError::Integrity); + } + hints.push(hint); + } + reader.finished()?; + Ok(hints) +} diff --git a/engine/crates/layerfs-persistence/src/blocks/mod.rs b/engine/crates/layerfs-persistence/src/blocks/mod.rs new file mode 100644 index 0000000000..178fbf367e --- /dev/null +++ b/engine/crates/layerfs-persistence/src/blocks/mod.rs @@ -0,0 +1,8 @@ +//! Append-only metadata blocks (#91, schema version 2): many inode values or +//! binding hints in one immutable row, field-packed so every entry expands +//! back to its exact canonical bytes. No compression: varints, implicit +//! keys and a per-block table of distinct modes only. Every block ends with +//! a CRC32C of the bytes before it, checked on every decode. +mod frame; +pub(crate) mod hints; +pub(crate) mod values; diff --git a/engine/crates/layerfs-persistence/src/blocks/values.rs b/engine/crates/layerfs-persistence/src/blocks/values.rs new file mode 100644 index 0000000000..e6fd03ab27 --- /dev/null +++ b/engine/crates/layerfs-persistence/src/blocks/values.rs @@ -0,0 +1,242 @@ +//! Inode value blocks: the values of consecutive ordinals in one row keyed +//! by its first ordinal; the ordinals inside are implicit. +//! +//! Body, format 1 (little-endian unless a varint): +//! +//! | Bytes | Field | +//! | --- | --- | +//! | 1 | format, `1` | +//! | 1 | entry count `n`, 1..=255: ordinals `first ..= first + n - 1` | +//! | 1 | distinct modes `m`, 0..=255 | +//! | 4 × m | the modes, each the value's four mode bytes as stored | +//! | … | `n` entries | +//! | 4 | CRC32C of every byte before it | +//! +//! An entry starts with a tag that is the value's byte 0 for the two +//! canonical forms (inode_leaf.rs): the root form (bit 7 clear) stores a +//! LEB128 reference count, the 32-byte content root and the 32-byte +//! attribute root; the inline form (bit 7 set, zero padding) stores the +//! count, the content root, a mode-table index, the mtime seconds as a +//! zig-zag LEB128 difference from the previous inline entry's (0 before the +//! first) and the four nanosecond bytes. Any other 73 bytes (and a byte 0 +//! of `0xff`) take tag `0xff` and are stored whole. Decoding rebuilds the +//! exact 73 bytes, so no object identity depends on this layout. +use super::frame::{self, Reader, BODY_LIMIT, CRC_BYTES}; +use crate::backend::records::BackendError; +use layerfs_content::inode_leaf::INODE_VALUE_BYTES; +use layerfs_storage::location::InodeValueRow; + +const FORMAT: u8 = 1; +const RAW: u8 = 0xff; +const HEADER: usize = 3; +const MAXIMUM_ENTRIES: usize = 255; +const MAXIMUM_MODES: usize = 255; + +/// One encoded block: ordinals `first ..= last`. +pub(crate) struct Block { + pub(crate) first: u32, + pub(crate) last: u32, + pub(crate) body: Vec, +} + +/// The block being filled. +struct Open { + first: u32, + count: usize, + modes: Vec<[u8; 4]>, + previous_mtime: i64, + entries: Vec, +} + +impl Open { + fn new(first: u32) -> Self { + Self { + first, + count: 0, + modes: Vec::new(), + previous_mtime: 0, + entries: Vec::with_capacity(BODY_LIMIT), + } + } + fn size(&self) -> usize { + HEADER + 4 * self.modes.len() + self.entries.len() + CRC_BYTES + } + /// Appends `value` when it fits; false leaves the block unchanged. + fn push(&mut self, value: &[u8; INODE_VALUE_BYTES]) -> bool { + if self.count == MAXIMUM_ENTRIES { + return false; + } + let inline = value[0] & 0x80 != 0 && value[0] != RAW && value[57..].iter().all(|b| *b == 0); + let root_form = value[0] & 0x80 == 0; + let mut entry = Vec::with_capacity(INODE_VALUE_BYTES + 1); + let mut new_mode = false; + let mut mtime = self.previous_mtime; + if inline || root_form { + entry.push(value[0]); + let count = u64::from_be_bytes(value[1..9].try_into().unwrap_or_default()); + frame::put_varint(&mut entry, count); + entry.extend_from_slice(&value[9..41]); + if root_form { + entry.extend_from_slice(&value[41..73]); + } else { + let mode: [u8; 4] = value[41..45].try_into().unwrap_or_default(); + let index = match self.modes.iter().position(|m| *m == mode) { + Some(index) => index, + None if self.modes.len() < MAXIMUM_MODES => { + new_mode = true; + self.modes.len() + } + None => return false, + }; + entry.push(index as u8); + mtime = i64::from_be_bytes(value[45..53].try_into().unwrap_or_default()); + frame::put_varint( + &mut entry, + frame::zigzag(mtime.wrapping_sub(self.previous_mtime)), + ); + entry.extend_from_slice(&value[53..57]); + } + } else { + entry.push(RAW); + entry.extend_from_slice(value); + } + let grown = self.size() + entry.len() + if new_mode { 4 } else { 0 }; + if self.count > 0 && grown > BODY_LIMIT { + return false; + } + if new_mode { + self.modes + .push(value[41..45].try_into().unwrap_or_default()); + } + self.previous_mtime = mtime; + self.entries.extend_from_slice(&entry); + self.count += 1; + true + } + fn finish(self) -> Block { + let mut body = Vec::with_capacity(self.size()); + body.push(FORMAT); + body.push(self.count as u8); + body.push(self.modes.len() as u8); + for mode in &self.modes { + body.extend_from_slice(mode); + } + body.extend_from_slice(&self.entries); + frame::seal(&mut body); + Block { + first: self.first, + last: self.first + (self.count as u32 - 1), + body, + } + } +} + +/// Packs distinct ascending ordinals' values into blocks, in order: a new +/// block starts at a gap in the ordinals or when the current one is full. +pub(crate) fn encode(rows: &[InodeValueRow]) -> Result, BackendError> { + let mut blocks = Vec::new(); + let mut open: Option = None; + let mut previous: Option = None; + for row in rows { + if row.ordinal == 0 || previous.is_some_and(|p| p >= row.ordinal) { + return Err(BackendError::Integrity); + } + let follows = previous.is_some_and(|p| p + 1 == row.ordinal); + previous = Some(row.ordinal); + if let Some(block) = open.as_mut() { + if follows && block.push(&row.value) { + continue; + } + } + if let Some(block) = open.take() { + blocks.push(block.finish()); + } + let mut block = Open::new(row.ordinal); + if !block.push(&row.value) { + return Err(BackendError::Integrity); + } + open = Some(block); + } + if let Some(block) = open { + blocks.push(block.finish()); + } + Ok(blocks) +} + +/// The values of ordinals `first ..= last`, from one block's body. +pub(crate) fn decode( + first: u32, + last: u32, + body: &[u8], +) -> Result, BackendError> { + if body.len() > BODY_LIMIT || last < first { + return Err(BackendError::Integrity); + } + let mut reader = Reader::new(frame::open(body)?); + let count = usize::from(reader.byte().and_then(|format| match format { + FORMAT => reader.byte(), + _ => Err(BackendError::Integrity), + })?); + if count == 0 || (last - first) as usize + 1 != count { + return Err(BackendError::Integrity); + } + let modes: Vec<&[u8]> = (0..reader.byte()?) + .map(|_| reader.take(4)) + .collect::>()?; + let mut previous_mtime = 0i64; + let mut values = Vec::with_capacity(count); + for _ in 0..count { + let mut value = [0u8; INODE_VALUE_BYTES]; + let tag = reader.byte()?; + if tag == RAW { + value.copy_from_slice(reader.take(INODE_VALUE_BYTES)?); + values.push(value); + continue; + } + value[0] = tag; + value[1..9].copy_from_slice(&reader.varint()?.to_be_bytes()); + value[9..41].copy_from_slice(reader.take(32)?); + if tag & 0x80 == 0 { + value[41..73].copy_from_slice(reader.take(32)?); + } else { + let mode = modes + .get(usize::from(reader.byte()?)) + .ok_or(BackendError::Integrity)?; + value[41..45].copy_from_slice(mode); + previous_mtime = previous_mtime.wrapping_add(frame::unzigzag(reader.varint()?)); + value[45..53].copy_from_slice(&previous_mtime.to_be_bytes()); + value[53..57].copy_from_slice(reader.take(4)?); + } + values.push(value); + } + reader.finished()?; + Ok(values) +} + +/// Picks the asked values out of blocks in ascending first-ordinal order, +/// in ordinal order; a block that overlaps the one before it is refused. +pub(crate) fn pick( + ordinals: &[u32], + blocks: impl IntoIterator), BackendError>>, + out: &mut Vec, +) -> Result<(), BackendError> { + let mut asked = ordinals.iter().copied().peekable(); + let mut end = 0u32; + for block in blocks { + let (first, last, body) = block?; + if first <= end { + return Err(BackendError::Integrity); + } + end = last; + let values = decode(first, last, &body)?; + while let Some(ordinal) = asked.next_if(|o| *o <= last) { + if ordinal >= first { + out.push(InodeValueRow { + ordinal, + value: values[(ordinal - first) as usize], + }); + } + } + } + Ok(()) +} diff --git a/engine/crates/layerfs-persistence/src/history/catalog.rs b/engine/crates/layerfs-persistence/src/history/catalog.rs index ec48745fbc..66fc5588b0 100644 --- a/engine/crates/layerfs-persistence/src/history/catalog.rs +++ b/engine/crates/layerfs-persistence/src/history/catalog.rs @@ -17,10 +17,10 @@ impl HistoryCatalog for HistoryProvider { self.read(|tx| hints::of(tx, child)) } fn record_file_digests(&self, rows: &[layerfs_history::FileDigest]) -> HistoryResult<()> { - self.write(|tx| super::digests::record(tx, rows)) + self.write(|tx| super::digests::record(tx, rows, self.digest_floor)) } fn file_roots(&self, contents: &[([u8; 32], u64)]) -> HistoryResult>> { - self.read_statement(|tx| super::digests::roots(tx, contents)) + super::digests::roots(self, contents) } fn catalog_id(&self) -> CatalogId { self.catalog_id diff --git a/engine/crates/layerfs-persistence/src/history/digests.rs b/engine/crates/layerfs-persistence/src/history/digests.rs index 9c5a84ffb8..b88f300122 100644 --- a/engine/crates/layerfs-persistence/src/history/digests.rs +++ b/engine/crates/layerfs-persistence/src/history/digests.rs @@ -1,7 +1,10 @@ -//! Saved file roots by the digest and length of their bytes (#364). +//! Saved file roots by the digest and length of their bytes (#364), kept +//! only for chunked files (#91): a file below the small-file threshold is one +//! whole-file object, so building it again finds the stored object by its +//! identity, and its digest row would cost more than it saves. use super::params; use super::rows::{cell, many, sql}; -use super::transaction::Transaction; +use super::{provider::HistoryProvider, transaction::Transaction}; use crate::backend::Column; use layerfs_history::error::{HistoryError, HistoryResult}; use layerfs_history::records::{FileDigest, FILE_DIGEST_WINDOW}; @@ -16,13 +19,19 @@ fn length(value: u64) -> HistoryResult { i64::try_from(value).map_err(|_| HistoryError::Integrity("file digest length")) } -/// Records the root built for each content; a recorded content takes the -/// newer root, so a row whose root the Store lost is replaced. +/// Records the root built for each content of at least `floor` bytes; a +/// recorded content takes the newer root, so a row whose root the Store lost +/// is replaced. A smaller content is not recorded. /// A repeated content keeps only its last root, as it would one row at a /// time; an engine with an array form then takes one statement per window, /// any other one statement per distinct content (a window of 1,000 empty /// files is one row, not 1,000). -pub(crate) fn record(tx: &Transaction<'_>, rows: &[FileDigest]) -> HistoryResult<()> { +pub(crate) fn record(tx: &Transaction<'_>, rows: &[FileDigest], floor: u64) -> HistoryResult<()> { + let rows: Vec = rows + .iter() + .filter(|row| row.length >= floor) + .copied() + .collect(); for part in rows.chunks(super::hints::ARRAY_ROWS) { let mut last = BTreeMap::new(); for row in part { @@ -59,8 +68,37 @@ pub(crate) fn record(tx: &Transaction<'_>, rows: &[FileDigest]) -> HistoryResult /// The recorded root of each (digest, length), in the order asked, when the /// Store holds that root's object: a row left by a Commit that never -/// finished names none. +/// finished names none. A content below the floor has no row and is not +/// asked for; a window of only such contents costs no statement. pub(crate) fn roots( + provider: &HistoryProvider, + contents: &[([u8; 32], u64)], +) -> HistoryResult>> { + if contents.len() > FILE_DIGEST_WINDOW { + return Err(HistoryError::Integrity("file digest window")); + } + let asked: Vec<([u8; 32], u64)> = contents + .iter() + .filter(|(_, length)| *length >= provider.digest_floor) + .copied() + .collect(); + let mut found = match asked.is_empty() { + true => Vec::new(), + false => provider.read_statement(|tx| stored(tx, &asked))?, + } + .into_iter(); + contents + .iter() + .map(|(_, length)| match *length >= provider.digest_floor { + true => found + .next() + .ok_or(HistoryError::Integrity("file digest roots")), + false => Ok(None), + }) + .collect() +} + +fn stored( tx: &Transaction<'_>, contents: &[([u8; 32], u64)], ) -> HistoryResult>> { diff --git a/engine/crates/layerfs-persistence/src/history/hints.rs b/engine/crates/layerfs-persistence/src/history/hints.rs index fc5725f1f1..4cb0690448 100644 --- a/engine/crates/layerfs-persistence/src/history/hints.rs +++ b/engine/crates/layerfs-persistence/src/history/hints.rs @@ -1,49 +1,56 @@ -//! Recorded name bindings of saved trees: hints for finding an inode's paths. +//! Recorded name bindings of saved trees: hints for finding an inode's +//! paths, kept in append-only blocks by group of 64 serials (#91). use super::params; use super::rows::{cell, many, sql}; use super::transaction::Transaction; use crate::backend::Column; -use layerfs_history::error::HistoryResult; +use crate::blocks::hints::{decode, encode, group_of}; +use layerfs_history::error::{HistoryError, HistoryResult}; use layerfs_history::records::BindingHint; -const INSERT_BINDING: &str = "hint_insert_binding"; -const INSERT_BINDINGS: &str = "hint_insert_bindings"; +const INSERT_BLOCK: &str = "hint_insert_block"; +const INSERT_BLOCKS: &str = "hint_insert_blocks"; /// Rows bound in one array statement. pub(crate) const ARRAY_ROWS: usize = 1024; -const BINDINGS_OF: &str = "hint_bindings_of"; +const BLOCKS_OF: &str = "hint_blocks_of"; -/// Records each binding once; a known row is left as it is. -/// An engine with an array form takes up to [`ARRAY_ROWS`] per statement. +/// Appends the bindings as new blocks, each the next of its group. A +/// binding recorded before may be recorded again: readers drop repeats. +/// The blocks are encoded before any statement; an engine with an array +/// form takes up to [`ARRAY_ROWS`] blocks per statement. pub(crate) fn record(tx: &Transaction<'_>, rows: &[BindingHint]) -> HistoryResult<()> { - for part in rows.chunks(ARRAY_ROWS) { + let blocks = + encode(rows).map_err(|_| HistoryError::Integrity("binding hint block encoding"))?; + for part in blocks.chunks(ARRAY_ROWS) { let columns = vec![ - Column::I64(part.iter().map(|row| row.child as i64).collect()), - Column::I64(part.iter().map(|row| row.parent as i64).collect()), - Column::Bytes(part.iter().map(|row| row.name.clone()).collect()), + Column::I64(part.iter().map(|block| block.group).collect()), + Column::Bytes(part.iter().map(|block| block.body.clone()).collect()), ]; - if let Some(done) = tx.arrays(INSERT_BINDINGS, columns) { + if let Some(done) = tx.arrays(INSERT_BLOCKS, columns) { done?; continue; } - for row in part { - tx.execute( - INSERT_BINDING, - params![row.child as i64, row.parent as i64, row.name.clone()], - ) - .map_err(sql)?; + for block in part { + tx.execute(INSERT_BLOCK, params![block.group, block.body.clone()]) + .map_err(sql)?; } } Ok(()) } -/// Every recorded binding of one inode serial. +/// Every recorded binding of one inode serial, in `(parent, name)` order +/// and without repeats, from its group's blocks. pub(crate) fn of(tx: &Transaction<'_>, child: u64) -> HistoryResult> { - many(tx, BINDINGS_OF, params![child as i64], |row| { - Ok(BindingHint { - child: cell::(row, 0)? as u64, - parent: cell::(row, 1)? as u64, - name: cell(row, 2)?, - }) - }) + let group = group_of(child); + let bodies = many(tx, BLOCKS_OF, params![group], |row| cell::>(row, 0))?; + let mut found = Vec::new(); + for body in bodies { + let hints = + decode(group, &body).map_err(|_| HistoryError::Integrity("binding hint block"))?; + found.extend(hints.into_iter().filter(|hint| hint.child == child)); + } + found.sort_unstable(); + found.dedup(); + Ok(found) } diff --git a/engine/crates/layerfs-persistence/src/history/provider.rs b/engine/crates/layerfs-persistence/src/history/provider.rs index 221ad1c6e7..b83a850351 100644 --- a/engine/crates/layerfs-persistence/src/history/provider.rs +++ b/engine/crates/layerfs-persistence/src/history/provider.rs @@ -7,4 +7,7 @@ pub struct HistoryProvider { pub(crate) catalog_id: CatalogId, pub(crate) incarnation: u64, pub(crate) cursor_key: [u8; 32], + /// The Store's small-file threshold: only files at or above it (the + /// chunked ones) keep a file digest (#91). + pub(crate) digest_floor: u64, } diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index 05166caa00..eb6b1df576 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -38,5 +38,6 @@ pub use store::postgresql::{ pub use store::seal::SealedStore; pub use store::selection::{Handles, StoreFile, StoreSelection}; +mod blocks; mod metadata; mod objects; diff --git a/engine/crates/layerfs-persistence/src/store/config.rs b/engine/crates/layerfs-persistence/src/store/config.rs index d500c0a367..e73fde9f54 100644 --- a/engine/crates/layerfs-persistence/src/store/config.rs +++ b/engine/crates/layerfs-persistence/src/store/config.rs @@ -111,33 +111,37 @@ pub enum SqlitePersistenceProfile { /// Explicit physical layout for a newly created embedded Store. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum SqlitePackLayout { - /// Original schema1 complete pack BLOBs. + /// Complete pack BLOBs (schema version 36, or 42 with acquisition tables). #[default] Monolithic, - /// Schema2 complete encoded groups in independent immutable rows. + /// Complete encoded groups in independent immutable rows (37 or 43). GroupRows, - /// Schema3 group rows with a covering index for bounded mapping validation. + /// Group rows with a covering index for bounded mapping validation (38 or 44). GroupRowsIndexed, } impl SqlitePackLayout { pub(crate) const fn uses_units(self) -> bool { matches!(self, Self::GroupRows | Self::GroupRowsIndexed) } + /// The stored schema version of this layout without acquisition + /// tables. Versions 36 to 38 (42 to 44 with them) are schema version 2 + /// (#91): inode values and binding hints in append-only blocks. The + /// version 1 layouts, 16 to 18 and 22 to 24, are refused at open. pub(crate) const fn version(self) -> i64 { match self { - Self::Monolithic => 16, - Self::GroupRows => 17, - Self::GroupRowsIndexed => 18, + Self::Monolithic => 36, + Self::GroupRows => 37, + Self::GroupRowsIndexed => 38, } } } /// Explicit creation-only selection of the initial-acquisition working tables. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum SqliteAcquisitionSchema { - /// Schema versions 16 to 18: no acquisition tables; acquisition is unavailable. + /// Schema versions 36 to 38: no acquisition tables; acquisition is unavailable. #[default] Absent, - /// Schema versions 22 to 24: the same pack layouts with the acquisition tables. + /// Schema versions 42 to 44: the same pack layouts with the acquisition tables. Tables, } impl SqliteAcquisitionSchema { diff --git a/engine/crates/layerfs-persistence/src/store/open.rs b/engine/crates/layerfs-persistence/src/store/open.rs index 4142605fcf..4e64058a22 100644 --- a/engine/crates/layerfs-persistence/src/store/open.rs +++ b/engine/crates/layerfs-persistence/src/store/open.rs @@ -101,7 +101,7 @@ impl SqliteHandles { return Err(PersistenceError::Malformed); } let catalog_id = CatalogId::derive(binding).map_err(|_| PersistenceError::Malformed)?; - let incarnation = session.validate(catalog_id, binding)?; + let (incarnation, digest_floor) = session.validate(catalog_id, binding)?; let storage = Arc::new(StorageProvider { session: session.clone(), }); @@ -113,6 +113,7 @@ impl SqliteHandles { catalog_id, incarnation, cursor_key, + digest_floor, }, }) } diff --git a/engine/crates/layerfs-persistence/src/store/postgresql.rs b/engine/crates/layerfs-persistence/src/store/postgresql.rs index 1569de7cae..5a961e3145 100644 --- a/engine/crates/layerfs-persistence/src/store/postgresql.rs +++ b/engine/crates/layerfs-persistence/src/store/postgresql.rs @@ -134,6 +134,7 @@ impl PostgresHandles { let checked = schema::check(&session, check, binding)?; let catalog_id = CatalogId::derive(&checked.binding).map_err(|_| PersistenceError::Malformed)?; + let digest_floor = checked.policy.small_file_threshold_bytes(); let _ = session.policy.set(checked.policy); let _ = session.server_version.set(checked.server_version); let (incarnation, tables) = (checked.incarnation, checked.acquisition); @@ -149,6 +150,7 @@ impl PostgresHandles { catalog_id, incarnation, cursor_key, + digest_floor, }, }, )) diff --git a/engine/crates/layerfs-persistence/tests/binding_hint_blocks.rs b/engine/crates/layerfs-persistence/tests/binding_hint_blocks.rs new file mode 100644 index 0000000000..bcb71ff048 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/binding_hint_blocks.rs @@ -0,0 +1,164 @@ +//! Binding hints in append-only blocks (#91): each write appends blocks of +//! at most 4,000 bytes per group of 64 serials, a reader merges a group's +//! blocks and drops repeats, and a hint no table row could hold is refused +//! before anything is written. +mod support; +use layerfs_history::{records::BindingHint, HistoryCatalog}; +use support::*; + +fn hint(child: u64, parent: u64, name: &str) -> BindingHint { + BindingHint { + child, + parent, + name: name.as_bytes().to_vec(), + } +} + +fn blocks(path: &std::path::Path) -> Vec<(i64, i64, usize)> { + let sql = rusqlite::Connection::open(path).unwrap(); + let mut statement = sql + .prepare("SELECT grp,seq,length(body) FROM binding_hint_block ORDER BY grp,seq") + .unwrap(); + statement + .query_map([], |r| { + Ok((r.get(0)?, r.get(1)?, r.get::<_, i64>(2)? as usize)) + }) + .unwrap() + .map(Result::unwrap) + .collect() +} + +#[test] +fn hints_are_merged_across_blocks_without_repeats() { + let t = Temp::new("hint-blocks"); + let path = t.join("db"); + let catalog = create(&path); + // Group 0 holds serials 1..=63; serial 64 starts group 1. + let mut first = vec![ + hint(7, 1, "b.txt"), + hint(7, 1, "a.txt"), + hint(7, 3, "a.txt"), + hint(63, 1, "last"), + hint(64, 1, "next-group"), + hint(7, 1, "a.txt"), + ]; + // Long names that share no prefix pass one block: the write appends + // several of one group. + for index in 0..40u64 { + first.push(hint(8 + index, 2, &format!("{index}-{}", "x".repeat(200)))); + } + catalog.record_bindings(&first).unwrap(); + let written = blocks(&path); + assert!(written.iter().all(|b| b.2 <= 4_000), "{written:?}"); + assert!( + written.iter().filter(|b| b.0 == 0).count() >= 2, + "{written:?}" + ); + assert_eq!(written.iter().filter(|b| b.0 == 1).count(), 1); + // A second write repeats one hint and adds one: a new block, no change + // to the stored ones, and the reader shows each binding once. + catalog + .record_bindings(&[hint(7, 1, "a.txt"), hint(7, 2, "c.txt")]) + .unwrap(); + let after = blocks(&path); + assert_eq!(after.len(), written.len() + 1); + assert!(written.iter().all(|block| after.contains(block))); + assert_eq!( + catalog.bindings_of(7).unwrap(), + vec![ + hint(7, 1, "a.txt"), + hint(7, 1, "b.txt"), + hint(7, 2, "c.txt"), + hint(7, 3, "a.txt"), + ] + ); + assert_eq!(catalog.bindings_of(63).unwrap(), vec![hint(63, 1, "last")]); + assert_eq!( + catalog.bindings_of(64).unwrap(), + vec![hint(64, 1, "next-group")] + ); + assert_eq!(catalog.bindings_of(47).unwrap().len(), 1); + assert!(catalog.bindings_of(65).unwrap().is_empty()); + assert!(catalog.bindings_of(1 << 40).unwrap().is_empty()); +} + +#[test] +fn a_hint_no_row_could_hold_is_refused_and_nothing_is_written() { + let t = Temp::new("hint-blocks-refused"); + let path = t.join("db"); + let catalog = create(&path); + for bad in [ + hint(0, 1, "a"), + hint(1, 0, "a"), + hint(1, 1, ""), + BindingHint { + child: 1, + parent: 1, + name: vec![b'n'; 256], + }, + ] { + assert!(catalog.record_bindings(&[hint(2, 1, "fine"), bad]).is_err()); + } + assert!(blocks(&path).is_empty()); + assert!(catalog.bindings_of(2).unwrap().is_empty()); +} + +#[test] +fn a_changed_block_fails_the_read_of_its_group() { + let t = Temp::new("hint-blocks-corrupt"); + let path = t.join("db"); + let catalog = create(&path); + catalog.record_bindings(&[hint(5, 1, "name")]).unwrap(); + let sql = rusqlite::Connection::open(&path).unwrap(); + let mut body: Vec = sql + .query_row("SELECT body FROM binding_hint_block", [], |r| r.get(0)) + .unwrap(); + body[3] ^= 0x10; + sql.execute("UPDATE binding_hint_block SET body=?1", [body]) + .unwrap(); + assert!(catalog.bindings_of(5).is_err()); + assert!(catalog.bindings_of(64).unwrap().is_empty()); +} + +/// The blocks of four fixed hints, byte for byte (see +/// `tools/test_store_blocks.py`). +#[test] +fn the_block_format_is_the_documented_one() { + let t = Temp::new("hint-blocks-golden"); + let path = t.join("db"); + let catalog = create(&path); + catalog + .record_bindings(&[ + hint(7, 1, "a.txt"), + hint(7, 1, "b.txt"), + hint(9, 2, "a.txt"), + hint(70, 1, "x"), + ]) + .unwrap(); + let sql = rusqlite::Connection::open(&path).unwrap(); + let mut statement = sql + .prepare("SELECT grp,seq,body FROM binding_hint_block ORDER BY grp") + .unwrap(); + let rows: Vec<(i64, i64, Vec)> = statement + .query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + let unhex = |hex: &str| -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() + }; + assert_eq!( + rows, + vec![ + ( + 0, + 1, + unhex("010307020005612e74787400000005622e74787402020005612e7478742d1e97ab") + ), + (1, 1, unhex("0101060200017860fbeefb")), + ] + ); +} diff --git a/engine/crates/layerfs-persistence/tests/old_format_open.rs b/engine/crates/layerfs-persistence/tests/old_format_open.rs index 88130806d5..7bcf056a27 100644 --- a/engine/crates/layerfs-persistence/tests/old_format_open.rs +++ b/engine/crates/layerfs-persistence/tests/old_format_open.rs @@ -1,4 +1,4 @@ -//! A Store written before the `binding_hint` table existed opens writable and +//! A Store written before the binding hint table existed opens writable and //! gains the table (#352 F-011, REBUILD.md kind 2): nothing it holds is //! rewritten, and hints are recorded from then on. mod support; @@ -14,7 +14,7 @@ const DIGESTS: &str = include_str!("../sql/sqlite/file_digest.sql"); /// The table's definition as the history schema writes it, with its trailing /// blank line: the text an old Store's recorded schema source lacks. fn hint_table() -> String { - let start = HISTORY.find("CREATE TABLE binding_hint (").unwrap(); + let start = HISTORY.find("CREATE TABLE binding_hint_block (").unwrap(); let end = start + HISTORY[start..].find(";\n").unwrap() + 2; format!("{}\n", &HISTORY[start..end]) } @@ -37,7 +37,7 @@ fn definition(sql: &rusqlite::Connection) -> String { fn make_old(path: &std::path::Path) { let sql = rusqlite::Connection::open(path).unwrap(); sql.execute_batch( - "DROP TABLE binding_hint; DROP TABLE layer_lineage; DROP TABLE commit_lineage; \ + "DROP TABLE binding_hint_block; DROP TABLE layer_lineage; DROP TABLE commit_lineage; \ DROP TABLE absorbed; DROP TABLE reach; DROP TABLE merge_operation; \ DROP TABLE merge_record; DROP TABLE file_digest;", ) diff --git a/engine/crates/layerfs-persistence/tests/postgresql_identical_ids.rs b/engine/crates/layerfs-persistence/tests/postgresql_identical_ids.rs index c41ac0fe85..765c29120a 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_identical_ids.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_identical_ids.rs @@ -412,27 +412,41 @@ fn hints_and_digests_in_arrays_read_back_as_on_sqlite() { } // Content 0 first names a root the Store lacks, then (past the first - // array of 1024 rows) the saved root: the later row wins. + // array of 1024 rows) the saved root: the later row wins. Only chunked + // files (at or above the small-file threshold) keep a digest (#91): the + // small content 2 is not recorded, so it has no root. + let floor = StoragePolicy::frozen_default().small_file_threshold_bytes(); let content = |i: u32| *ObjectId::for_bytes(&i.to_le_bytes()).as_bytes(); let missing = *root(77).as_bytes(); let mut digests: Vec = (0..1100u32) .map(|i| FileDigest { root: missing, - length: u64::from(i), + length: floor + u64::from(i), digest: content(i), }) .collect(); digests[1050] = FileDigest { root: *saved.as_bytes(), - length: 0, + length: floor, digest: content(0), }; - let asked = [(content(0), 0), (content(1), 1), (content(5000), 5000)]; + digests.push(FileDigest { + root: *saved.as_bytes(), + length: floor - 1, + digest: content(2), + }); + let asked = [ + (content(0), floor), + (content(1), floor + 1), + (content(2), floor - 1), + (content(5000), floor + 5000), + ]; for engine in engines { engine.record_file_digests(&digests).unwrap(); assert_eq!( engine.file_roots(&asked).unwrap(), - vec![Some(*saved.as_bytes()), None, None] + vec![Some(*saved.as_bytes()), None, None, None] ); + assert_eq!(engine.file_roots(&asked[2..3]).unwrap(), vec![None]); } } diff --git a/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs b/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs new file mode 100644 index 0000000000..4e3bdb10a4 --- /dev/null +++ b/engine/crates/layerfs-persistence/tests/postgresql_narrow_locations.rs @@ -0,0 +1,247 @@ +//! #90: PostgreSQL object locations are narrow rows found through a B-tree +//! on the id's first eight bytes; the full id is compared, and uniqueness is +//! kept under the lane by inserting only absent ids. Ids that share their +//! first eight bytes are distinct objects; an id already stored is lost to +//! the first writer, never stored twice; a signature naming no location is +//! refused with everything else of its publication; the tiny tables have no +//! TOAST relation. +//! +//! Needs `LAYERFS_TEST_POSTGRES_URL`; NOT_RUN without it. +#![cfg(feature = "postgresql")] +use layerfs_content::{ObjectId, ObjectRole}; +use layerfs_history::HistoryCatalogConfig; +use layerfs_persistence::{PostgresHandles, PostgresSelection}; +use layerfs_storage::{ + location::{ObjectLocation, PackDomain, PackInfo, SignatureRow}, + pack::{assemble, build_group, PackLane}, + port::*, + StoragePolicy, +}; +use std::sync::Arc; + +struct Schema { + url: String, + selection: PostgresSelection, +} +impl Schema { + fn new(url: String, tag: &str) -> (Self, PostgresHandles) { + let name = format!("lfs90_{tag}_{}", std::process::id()); + let schema = Self { + selection: PostgresSelection::new(&url, &name), + url, + }; + schema.sql(&format!("DROP SCHEMA IF EXISTS {name} CASCADE")); + let handles = PostgresHandles::create( + &schema.selection, + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: b"postgresql-narrow-locations".to_vec(), + cursor_key: [90; 32], + incarnation: 5, + }, + ) + .unwrap(); + (schema, handles) + } + fn sql(&self, sql: &str) { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).unwrap(); + client.batch_execute(sql).unwrap(); + } + fn count(&self, sql: &str) -> i64 { + let mut client = postgres::Client::connect(&self.url, postgres::NoTls).unwrap(); + client.query_one(sql, &[]).unwrap().get(0) + } +} +impl Drop for Schema { + fn drop(&mut self) { + let _ = postgres::Client::connect(&self.url, postgres::NoTls).and_then(|mut c| { + c.batch_execute(&format!( + "DROP SCHEMA IF EXISTS {} CASCADE", + self.selection.schema + )) + }); + } +} +fn url() -> Option { + let url = std::env::var("LAYERFS_TEST_POSTGRES_URL").ok(); + if url.is_none() { + eprintln!("NOT_RUN: LAYERFS_TEST_POSTGRES_URL is unset"); + } + url +} + +/// An id whose first eight bytes are `head` and whose rest is `tail`. +fn id(head: u8, tail: u8) -> ObjectId { + let mut bytes = [tail; 32]; + bytes[..8].copy_from_slice(&[head; 8]); + ObjectId::from_bytes(&bytes).unwrap() +} +fn location(object_id: ObjectId, pack_id: i64, record_number: usize) -> ObjectLocation { + ObjectLocation { + object_id, + role: ObjectRole::WholeFile, + canonical_length: 64, + pack_id, + group_number: 0, + record_number, + } +} +/// A publication of one new pack holding `objects`. +fn unit(handles: &PostgresHandles, objects: Vec<(ObjectId, usize)>) -> Publication { + let pack_id = handles + .storage + .reserve(Reserve { + packs: 1, + ordinals: 0, + }) + .unwrap() + .first_pack_id; + let body = assemble( + PackLane::WholeFile, + &[build_group(PackLane::WholeFile, &[vec![0x10; 64]], None).unwrap()], + ) + .unwrap(); + Publication { + packs: vec![PublishedPack { + info: PackInfo { + pack_id, + domain: PackDomain::Payload, + key: ObjectKey::for_bytes(&body), + length: body.len(), + }, + body: Arc::new(body), + }], + objects: objects + .into_iter() + .map(|(object, record)| location(object, pack_id, record)) + .collect(), + ..Publication::default() + } +} +fn located(handles: &PostgresHandles, ids: &[ObjectId]) -> Vec<(ObjectId, i64, usize)> { + let mut out = Vec::new(); + handles.storage.locate(ids, &mut out).unwrap(); + let mut found: Vec<_> = out + .into_iter() + .map(|o| { + ( + o.location.object_id, + o.location.pack_id, + o.location.record_number, + ) + }) + .collect(); + found.sort(); + found +} + +#[test] +fn ids_that_share_a_prefix_are_distinct_and_a_repeat_is_lost_to_the_first() { + let Some(url) = url() else { return }; + let (schema, handles) = Schema::new(url, "prefix"); + let s = &schema.selection.schema; + // Three ids with one prefix, one with another. + let (a, b, c, d) = (id(1, 1), id(1, 2), id(1, 3), id(2, 1)); + // A batch that repeats an id is refused before anything is sent. + let repeated = unit(&handles, vec![(a, 0), (a, 1)]); + assert_eq!( + handles.storage.publish(&repeated).unwrap_err(), + PersistenceError::Malformed + ); + let first = unit(&handles, vec![(a, 0), (b, 1), (d, 2)]); + let one = first.packs[0].info.pack_id; + assert_eq!(handles.storage.publish(&first).unwrap().lost, vec![]); + assert_eq!( + located(&handles, &[a, b, c, d]), + vec![(a, one, 0), (b, one, 1), (d, one, 2)] + ); + // A second writer publishes a stored id and a new one with the same + // prefix: the stored one is lost and keeps its location. + let second = unit(&handles, vec![(b, 4), (c, 5)]); + let two = second.packs[0].info.pack_id; + assert_eq!(handles.storage.publish(&second).unwrap().lost, vec![b]); + assert_eq!( + located(&handles, &[a, b, c, d]), + vec![(a, one, 0), (b, one, 1), (c, two, 5), (d, one, 2)] + ); + assert_eq!( + schema.count(&format!("SELECT count(*) FROM {s}.object_location")), + 4 + ); + // The lookup goes through the prefix index, not a scan. + let mut client = postgres::Client::connect(&schema.url, postgres::NoTls).unwrap(); + let plan: Vec = client + .query( + &format!( + "EXPLAIN SELECT 1 FROM {s}.object_location o WHERE \ + ('x'||encode(substring(o.object_id FROM 1 FOR 8),'hex'))::bit(64)::bigint=$1 AND o.object_id=$2" + ), + &[&0x0101_0101_0101_0101_i64, &a.as_bytes().as_slice()], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert!( + plan.iter() + .any(|line| line.contains("object_location_prefix")), + "{plan:?}" + ); +} + +#[test] +fn a_signature_that_names_no_location_refuses_its_publication() { + let Some(url) = url() else { return }; + let (schema, handles) = Schema::new(url, "signature"); + let s = &schema.selection.schema; + let stored = id(3, 1); + handles + .storage + .publish(&unit(&handles, vec![(stored, 0)])) + .unwrap(); + let signature = |slot: usize, object_id| SignatureRow { + slot, + stamp: slot as u64 + 1, + object_id, + signature: [slot as u8; 32], + }; + // A signature may name a stored location or one of its publication. + let fresh = id(3, 2); + let mut batch = unit(&handles, vec![(fresh, 0)]); + batch.signatures = vec![signature(1, stored), signature(2, fresh)]; + handles.storage.publish(&batch).unwrap(); + // One that names neither refuses the whole publication. + let other = id(3, 3); + let mut refused = unit(&handles, vec![(other, 0)]); + refused.signatures = vec![signature(3, id(3, 4))]; + assert!(handles.storage.publish(&refused).is_err()); + assert_eq!(located(&handles, &[other]), vec![]); + let mut rows = Vec::new(); + handles.storage.signatures(&mut rows).unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!( + schema.count(&format!("SELECT count(*) FROM {s}.object_location")), + 2 + ); +} + +#[test] +fn only_tables_with_unbounded_values_have_a_toast_relation() { + let Some(url) = url() else { return }; + let (schema, _handles) = Schema::new(url, "toast"); + let mut client = postgres::Client::connect(&schema.url, postgres::NoTls).unwrap(); + let toasted: Vec = client + .query( + "SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relkind='r' AND c.reltoastrelid<>0 ORDER BY 1", + &[&schema.selection.schema], + ) + .unwrap() + .iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + toasted, + ["commit", "history_meta", "layer", "pack", "reach"] + ); +} diff --git a/engine/crates/layerfs-persistence/tests/postgresql_open_check.rs b/engine/crates/layerfs-persistence/tests/postgresql_open_check.rs index d05b1df60a..ba7c1e3ea2 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_open_check.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_open_check.rs @@ -112,13 +112,14 @@ fn both_checks_refuse_a_changed_incarnation_or_schema_version() { ); } schema.sql(&format!( - "UPDATE {s}.history_meta SET catalog_incarnation=3, schema_version=2" + "UPDATE {s}.history_meta SET catalog_incarnation=3, schema_version=1" )); for check in CHECKS { + // a schema version 1 schema (rows, before #91's blocks) is refused assert!(schema.open(check, BINDING).is_err(), "{check:?}: version"); } schema.sql(&format!( - "UPDATE {s}.history_meta SET schema_version=1, identity_format=2" + "UPDATE {s}.history_meta SET schema_version=2, identity_format=2" )); for check in CHECKS { assert!(schema.open(check, BINDING).is_err(), "{check:?}: format"); @@ -146,6 +147,19 @@ fn only_the_full_check_reads_the_live_definition() { schema.sql(&format!("ALTER TABLE {s}.reach ADD COLUMN extra bigint")); assert!(schema.open(PostgresCheck::Full, BINDING).is_err()); schema.open(PostgresCheck::Light, BINDING).unwrap(); + // a version 1 wide location column (#90 narrowed `role` to smallint): + // only the full check sees the type + schema.sql(&format!( + "ALTER TABLE {s}.reach DROP COLUMN extra; \ + ALTER TABLE {s}.object_location ALTER COLUMN role TYPE bigint" + )); + assert!(schema.open(PostgresCheck::Full, BINDING).is_err()); + schema.open(PostgresCheck::Light, BINDING).unwrap(); + schema.sql(&format!( + "ALTER TABLE {s}.object_location ALTER COLUMN role TYPE smallint; \ + ALTER TABLE {s}.reach ADD COLUMN extra bigint" + )); + schema.open(PostgresCheck::Light, BINDING).unwrap(); // an acquisition table gone: the full check sees the table set, the // light one the three names, so both refuse schema.sql(&format!( diff --git a/engine/crates/layerfs-persistence/tests/sqlite_group_rows.rs b/engine/crates/layerfs-persistence/tests/sqlite_group_rows.rs index cd3dd5ff35..0b770608fd 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_group_rows.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_group_rows.rs @@ -96,7 +96,7 @@ fn explicit_creation_reopens_declared_units_and_keeps_sparse_canonical_order() { assert_eq!( db.query_row("PRAGMA user_version", [], |r| r.get::<_, i64>(0)) .unwrap(), - 17 + 37 ); assert_eq!( db.query_row("SELECT count(*) FROM pack_unit", [], |r| r.get::<_, i64>(0)) diff --git a/engine/crates/layerfs-persistence/tests/sqlite_group_rows_indexed.rs b/engine/crates/layerfs-persistence/tests/sqlite_group_rows_indexed.rs index 58449bdfff..66ef8d3cbf 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_group_rows_indexed.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_group_rows_indexed.rs @@ -88,9 +88,9 @@ fn fixture(h: &SqliteHandles) -> Vec { #[test] fn declared_layouts_reopen_without_mutating_or_promoting_their_schema() { for (layout, version) in [ - (SqlitePackLayout::Monolithic, 16), - (SqlitePackLayout::GroupRows, 17), - (SqlitePackLayout::GroupRowsIndexed, 18), + (SqlitePackLayout::Monolithic, 36), + (SqlitePackLayout::GroupRows, 37), + (SqlitePackLayout::GroupRowsIndexed, 38), ] { let t = support::Temp::new("indexed-legacy-open"); let path = t.join("db"); diff --git a/engine/crates/layerfs-persistence/tests/sqlite_inode_values.rs b/engine/crates/layerfs-persistence/tests/sqlite_inode_values.rs index 22c2138100..08e8ce0aff 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_inode_values.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_inode_values.rs @@ -1,6 +1,7 @@ -//! The stored inode value table: one row an ordinal, read in one statement -//! execution, written with the Save's own publication, never replaced, and -//! refused rather than misread when the Store predates it. +//! The stored inode value blocks (#91): many consecutive ordinals' values in +//! one row, read in one statement execution, written with the Save's own +//! publication, never replaced, every 73-byte value back exactly, and +//! refused rather than misread when the Store predates them. mod support; use layerfs_content::{ inode_leaf::{ @@ -184,30 +185,29 @@ fn a_row_changed_in_the_file_fails_the_read_of_its_leaf() { }; assert_eq!(read().unwrap(), vec![object.canonical().to_vec()]); let db = rusqlite::Connection::open(&path).unwrap(); - let stored: Vec<(i64, Vec)> = db - .prepare("SELECT ordinal,value FROM inode_value ORDER BY ordinal") + let stored: Vec<(i64, i64, Vec)> = db + .prepare("SELECT first_ordinal,last_ordinal,body FROM inode_value_block ORDER BY 1") .unwrap() - .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) .unwrap() .map(Result::unwrap) .collect(); - assert_eq!(stored.len(), 4); - assert!(stored.iter().all(|(_, value)| value.len() == 73)); - // A well-formed value of the right length that is not the one saved. + // The four values are one block. + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].1 - stored[0].0, 3); + // One changed byte of the block fails its checksum. + let mut body = stored[0].2.clone(); + body[40] ^= 1; db.execute( - "UPDATE inode_value SET value=?1 WHERE ordinal=?2", - rusqlite::params![value(77).to_vec(), stored[2].0], + "UPDATE inode_value_block SET body=?1 WHERE first_ordinal=?2", + rusqlite::params![body, stored[0].0], ) .unwrap(); - let error = read().unwrap_err(); - assert!( - error.to_string().contains("pooled canonical identity"), - "{error}" - ); - // The same for a deleted row. + assert!(read().is_err()); + // A deleted block leaves its ordinals missing. db.execute( - "DELETE FROM inode_value WHERE ordinal=?1", - rusqlite::params![stored[2].0], + "DELETE FROM inode_value_block WHERE first_ordinal=?1", + rusqlite::params![stored[0].0], ) .unwrap(); let error = read().unwrap_err(); @@ -215,13 +215,95 @@ fn a_row_changed_in_the_file_fails_the_read_of_its_leaf() { error.to_string().contains("metadata ordinal missing"), "{error}" ); - // The table itself refuses a value of another length. - assert!(db - .execute( - "INSERT INTO inode_value(ordinal,value) VALUES(?1,?2)", - rusqlite::params![stored[2].0, vec![0u8; 72]], - ) - .is_err()); + // The table itself refuses a body past one block and a run past 255. + for (last, body) in [ + (stored[0].1, vec![1u8; 4001]), + (stored[0].0 + 255, vec![1u8; 64]), + ] { + assert!(db + .execute( + "INSERT INTO inode_value_block(first_ordinal,last_ordinal,body) VALUES(?1,?2,?3)", + rusqlite::params![stored[0].0, last, body], + ) + .is_err()); + } +} + +/// Values of every shape: both canonical forms, inline values with nonzero +/// padding and other bytes no grammar accepts, all of which come back +/// byte for byte. +fn odd_value(seed: u32) -> [u8; 73] { + let mut value = [0u8; 73]; + for (index, byte) in value.iter_mut().enumerate() { + *byte = (seed as usize * 31 + index * 7) as u8; + } + match seed % 6 { + 0 => value[0] = 0xff, + 1 => { + value[0] = 0x81; + value[57..].fill(0); + value[1..9].copy_from_slice(&u64::MAX.to_be_bytes()); + value[45..53].copy_from_slice(&i64::MIN.to_be_bytes()); + } + 2 => { + value[0] = 0x82; + value[57..].fill(0); + value[45..53].copy_from_slice(&i64::MAX.to_be_bytes()); + } + 3 => value[0] = 0x83, // nonzero padding: stored whole + 4 => value[0] = 0x05, + _ => { + value[0] = 0x81; + value[57..].fill(0); + value[41..45].copy_from_slice(&seed.to_be_bytes()); + } + } + value +} + +#[test] +fn every_value_comes_back_exactly_from_blocks_of_at_most_4000_bytes() { + let temp = Temp::new("inode-values-exact"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + let stored: Vec = (1..=700) + .chain(900..=1_200) + .map(|ordinal| InodeValueRow { + ordinal, + value: if ordinal % 2 == 0 { + odd_value(ordinal) + } else { + value(u64::from(ordinal)) + }, + }) + .collect(); + publish(&handles, stored.clone()).unwrap(); + let wanted: Vec = stored.iter().map(|row| row.ordinal).collect(); + let mut found = Vec::new(); + for part in wanted.chunks(512) { + let mut some = Vec::new(); + handles.storage.inode_values(part, &mut some).unwrap(); + found.extend(some); + } + assert_eq!(found, stored); + let db = rusqlite::Connection::open(&path).unwrap(); + let blocks: Vec<(i64, i64, i64)> = db + .prepare("SELECT first_ordinal,last_ordinal,length(body) FROM inode_value_block ORDER BY 1") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .unwrap() + .map(Result::unwrap) + .collect(); + // No block spans the gap, none is past 4,000 bytes, and together they + // hold every ordinal once. + assert!(blocks + .iter() + .all(|b| b.2 <= 4_000 && (b.1 < 900 || b.0 >= 900))); + assert_eq!( + blocks.iter().map(|b| b.1 - b.0 + 1).sum::(), + stored.len() as i64 + ); + assert!(blocks.len() < stored.len() / 40, "{}", blocks.len()); } #[test] @@ -230,15 +312,39 @@ fn a_store_of_the_value_group_format_is_refused_at_open() { // `metadata_value_group`. Neither the old version number nor the old // table is accepted, with or without the other. Versions 7 and 13 are // the formats before #327's commit-v3 records, refused without migration. + // Versions 16 to 18 and 22 to 24 are schema version 1, one row per + // value, refused since #91 without conversion. + // Schema version 1 is refused with the offline conversion named. + for version in [16, 17, 18, 22, 23, 24] { + let temp = Temp::new("inode-values-v1-store"); + let path = temp.join("store.sqlite"); + drop(create(&path)); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch(&format!("PRAGMA user_version={version}")) + .unwrap(); + for opened in [ + SqliteHandles::open_writable(config(&path), BINDING, [73; 32]).err(), + SqliteHandles::open_read_only(config(&path), BINDING, [73; 32]).err(), + ] { + match opened { + Some(PersistenceError::Refused { status }) => { + assert!(status.contains("tools/store_blocks.py migrate"), "{status}") + } + other => panic!("{version}: {other:?}"), + } + } + } for change in [ + "CREATE TABLE inode_value (ordinal INTEGER PRIMARY KEY) STRICT", "PRAGMA user_version=1", "PRAGMA user_version=4", "PRAGMA user_version=6", "PRAGMA user_version=7", "PRAGMA user_version=13", "CREATE TABLE metadata_value_group (first_ordinal INTEGER PRIMARY KEY) STRICT", - "DROP TABLE inode_value", - "DROP TABLE inode_value; CREATE TABLE metadata_value_group (first_ordinal INTEGER PRIMARY KEY) STRICT; PRAGMA user_version=1", + "DROP TABLE inode_value_block", + "DROP TABLE inode_value_block; CREATE TABLE metadata_value_group (first_ordinal INTEGER PRIMARY KEY) STRICT; PRAGMA user_version=1", ] { let temp = Temp::new("inode-values-old-store"); let path = temp.join("store.sqlite"); @@ -262,6 +368,64 @@ fn a_store_of_the_value_group_format_is_refused_at_open() { .unwrap() .query_row("PRAGMA user_version", [], |row| row.get(0)) .unwrap(); - assert_eq!(version, 16); + assert_eq!(version, 36); SqliteHandles::open_writable(config(&path), BINDING, [73; 32]).unwrap(); } + +/// The block of four fixed values, byte for byte: the vector +/// `tools/test_store_blocks.py` decodes and encodes to the same bytes, so the +/// diagnostic decoder and the offline migration agree with the engine. +const GOLDEN_VALUES: &str = "010402000081a4000081ed810111111111111111111111111111111111111111111111111111111111111111110080c49fd50c0102030481011111111111111111111111111111111111111111111111111111111111111111010a01020304010222222222222222222222222222222222222222222222222222222222222222223333333333333333333333333333333333333333333333333333333333333333fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff2f3427f"; + +fn golden_values() -> Vec { + let inline = |mode: u32, mtime: i64| { + let mut value = [0u8; 73]; + value[0] = 0x81; + value[1..9].copy_from_slice(&1u64.to_be_bytes()); + value[9..41].fill(0x11); + value[41..45].copy_from_slice(&mode.to_be_bytes()); + value[45..53].copy_from_slice(&mtime.to_be_bytes()); + value[53..57].copy_from_slice(&[1, 2, 3, 4]); + value + }; + let mut root = [0u8; 73]; + root[0] = 1; + root[1..9].copy_from_slice(&2u64.to_be_bytes()); + root[9..41].fill(0x22); + root[41..73].fill(0x33); + [ + inline(0o100644, 1_700_000_000), + inline(0o100755, 1_700_000_005), + root, + [0xff; 73], + ] + .into_iter() + .zip(5..) + .map(|(value, ordinal)| InodeValueRow { ordinal, value }) + .collect() +} + +#[test] +fn the_block_format_is_the_documented_one() { + let temp = Temp::new("inode-values-golden"); + let path = temp.join("store.sqlite"); + let handles = create(&path); + publish(&handles, golden_values()).unwrap(); + let (first, last, body): (i64, i64, Vec) = rusqlite::Connection::open(&path) + .unwrap() + .query_row( + "SELECT first_ordinal,last_ordinal,body FROM inode_value_block", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!((first, last), (5, 8)); + assert_eq!(body, unhex(GOLDEN_VALUES)); +} + +fn unhex(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() +} diff --git a/engine/crates/layerfs-persistence/tests/sqlite_ordinal_lookahead.rs b/engine/crates/layerfs-persistence/tests/sqlite_ordinal_lookahead.rs index 8d17dcdeb4..d424d9c04c 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_ordinal_lookahead.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_ordinal_lookahead.rs @@ -88,12 +88,15 @@ fn fresh_leaves_share_one_combined_reservation_and_deduplicate_values() { .unwrap(); assert_eq!(allocator, 6); let ordinals = sql - .prepare("SELECT ordinal FROM inode_value ORDER BY ordinal") + .prepare("SELECT first_ordinal,last_ordinal FROM inode_value_block ORDER BY 1") .unwrap() - .query_map([], |r| r.get::<_, i64>(0)) + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?))) .unwrap() .collect::, _>>() - .unwrap(); + .unwrap() + .into_iter() + .flat_map(|(first, last)| first..=last) + .collect::>(); assert_eq!(ordinals, vec![1, 2, 3, 4, 5]); let before = storage.diagnostics().ordinal_reservations; let duplicate = leaf(20, &[1, 2, 3, 4, 5]); @@ -130,7 +133,11 @@ fn a_foreign_acknowledged_reservation_is_never_recycled_by_the_plan() { read(&storage, &objects); let sql = rusqlite::Connection::open(t.join("db")).unwrap(); let first: i64 = sql - .query_row("SELECT min(ordinal) FROM inode_value", [], |r| r.get(0)) + .query_row( + "SELECT min(first_ordinal) FROM inode_value_block", + [], + |r| r.get(0), + ) .unwrap(); assert_eq!(first, 8); let next: i64 = sql @@ -215,7 +222,11 @@ fn initial_combined_reservation_is_busy_before_effect() { .unwrap(); assert_eq!(first, 1, "Busy consumed no pack ids"); let first: i64 = sql - .query_row("SELECT min(ordinal) FROM inode_value", [], |row| row.get(0)) + .query_row( + "SELECT min(first_ordinal) FROM inode_value_block", + [], + |row| row.get(0), + ) .unwrap(); assert_eq!(first, 1, "Busy consumed no value ordinals"); println!("SAVE_BLOCK_BUSY initial_attempts=2 busy_effect=0 later_explicit_save=complete first_pack=1 first_ordinal=1 child=released-and-joined"); diff --git a/engine/crates/layerfs-store-service/tests/fixtures/merge-forms-root.sqlite b/engine/crates/layerfs-store-service/tests/fixtures/merge-forms-root.sqlite index 402cfa2468..3615ffda40 100644 Binary files a/engine/crates/layerfs-store-service/tests/fixtures/merge-forms-root.sqlite and b/engine/crates/layerfs-store-service/tests/fixtures/merge-forms-root.sqlite differ diff --git a/engine/crates/layerfs-store-service/tests/merge_forms_fixture.rs b/engine/crates/layerfs-store-service/tests/merge_forms_fixture.rs index eb71dfdfa4..e7e55f11ea 100644 --- a/engine/crates/layerfs-store-service/tests/merge_forms_fixture.rs +++ b/engine/crates/layerfs-store-service/tests/merge_forms_fixture.rs @@ -4,7 +4,8 @@ //! form (a move, two content edits and a mode change), for a build with the //! inline form to merge against. The output is //! checked in at `tests/fixtures/merge-forms-root.sqlite` on the branch that -//! carries #360. +//! carries #360, and converted to schema version 2 (#91, inode value and +//! binding hint blocks) with `tools/store_blocks.py migrate`. #[path = "support/installed_store.rs"] #[allow(dead_code)] mod installed; diff --git a/tools/store_blocks.py b/tools/store_blocks.py new file mode 100755 index 0000000000..51ac08082d --- /dev/null +++ b/tools/store_blocks.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""Inode value and binding hint blocks of a schema version 2 Store (#91). + +Diagnostic decoder and offline SQLite migration; never run against a live Store. + + store_blocks.py attribute + Decodes every block of a closed SQLite copy (opened immutable, read only) + and prints per-table entry, block and byte counts with a per-field split, + as JSON. DIAGNOSTIC: rows are no longer readable with plain SQL. + store_blocks.py attribute --rows + The same for rows exported from a PostgreSQL schema, one per line: + `valuefirstlasthex body` or `hintgrpseqhex body` + (for example `psql -At -F $'\\t' -c "SELECT 'value',first_ordinal,last_ordinal, + encode(body,'hex') FROM .inode_value_block"`). + store_blocks.py migrate + Writes a schema version 2 copy of a closed schema version 1 SQLite Store + (user_version 16-18 or 22-24): inode values and binding hints become blocks, + the version moves to 36-38 or 42-44 and the recorded schema is rewritten. + The source is copied first and never changed; a Store whose recorded schema + is not exactly one this release wrote (allowing the tables the open-time + additive migrations add) is refused. PostgreSQL schemas are not migrated. + +The block formats are documented in engine/crates/layerfs-persistence/src/blocks/. +""" + +import json +import os +import shutil +import sqlite3 +import sys +from collections import Counter, defaultdict + +BODY_LIMIT = 4000 +VALUE_BYTES = 73 +RAW = 0xFF +GROUP_BITS = 6 +APPLICATION_ID = 1279677264 +SQL = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "engine", "crates", "layerfs-persistence", "sql") + +# Schema version 1 table texts, as release 2.0 wrote them before #91. +V1_VALUES = """CREATE TABLE inode_value ( + ordinal INTEGER PRIMARY KEY CHECK(ordinal BETWEEN 1 AND 4294967295), + value BLOB NOT NULL CHECK(length(value)=73) +) STRICT;""" +V1_HINTS = """CREATE TABLE binding_hint ( + child INTEGER NOT NULL CHECK (child != 0), + parent INTEGER NOT NULL CHECK (parent != 0), + name BLOB NOT NULL CHECK (length(name) BETWEEN 1 AND 255), + PRIMARY KEY (child, parent, name) +) STRICT, WITHOUT ROWID;""" + + +def _crc_table(): + table = [] + for byte in range(256): + crc = byte + for _ in range(8): + crc = (crc >> 1) ^ 0x82F63B78 if crc & 1 else crc >> 1 + table.append(crc) + return table + + +CRC_TABLE = _crc_table() + + +def crc32c(data): + crc = 0xFFFFFFFF + for byte in data: + crc = CRC_TABLE[(crc ^ byte) & 0xFF] ^ (crc >> 8) + return crc ^ 0xFFFFFFFF + + +def varint(value): + out = bytearray() + while value >= 0x80: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + return bytes(out) + + +def zigzag(value): + value = ((value + 2**63) % 2**64) - 2**63 # wrap to i64 + return ((value << 1) ^ (value >> 63)) & (2**64 - 1) + + +def unzigzag(value): + return (value >> 1) ^ -(value & 1) + + +def wrap64(value): + return ((value + 2**63) % 2**64) - 2**63 + + +class Reader: + def __init__(self, data): + self.data, self.at = data, 0 + + def take(self, n): + if self.at + n > len(self.data): + raise ValueError("block truncated") + part = self.data[self.at : self.at + n] + self.at += n + return part + + def byte(self): + return self.take(1)[0] + + def varint(self): + value = 0 + for index in range(10): + byte = self.byte() + if (index == 9 and byte > 1) or (index > 0 and byte == 0): + raise ValueError("non-minimal varint") + value |= (byte & 0x7F) << (7 * index) + if not byte & 0x80: + return value + raise ValueError("varint too long") + + +def opened(body): + if len(body) < 4 or len(body) > BODY_LIMIT: + raise ValueError("block length") + content, trailer = body[:-4], body[-4:] + if crc32c(content) != int.from_bytes(trailer, "little"): + raise ValueError("block checksum") + return content + + +def _value_entry(value, modes, previous_mtime): + """(entry bytes, new mode or None, mtime) of one value; modes is the block's table.""" + inline = value[0] & 0x80 and value[0] != RAW and not any(value[57:]) + if not inline and value[0] & 0x80: + return bytes([RAW]) + value, None, previous_mtime + entry = bytes([value[0]]) + varint(int.from_bytes(value[1:9], "big")) + value[9:41] + if not inline: + return entry + value[41:73], None, previous_mtime + mode = value[41:45] + new = None + if mode in modes: + index = modes.index(mode) + elif len(modes) < 255: + index, new = len(modes), mode + else: + return None, None, previous_mtime + mtime = int.from_bytes(value[45:53], "big", signed=True) + entry += bytes([index]) + varint(zigzag(mtime - previous_mtime)) + value[53:57] + return entry, new, mtime + + +def encode_values(rows): + """[(first, last, body)] of ascending distinct (ordinal, 73 bytes) rows, as the engine packs them.""" + blocks, block, previous = [], None, None + + def finish(b): + body = bytes([1, b["count"], len(b["modes"])]) + b"".join(b["modes"]) + bytes(b["entries"]) + body += crc32c(body).to_bytes(4, "little") + blocks.append((b["first"], b["first"] + b["count"] - 1, body)) + + def push(b, value): + if b["count"] == 255: + return False + entry, new, mtime = _value_entry(value, b["modes"], b["mtime"]) + if entry is None: + return False + size = 3 + 4 * len(b["modes"]) + len(b["entries"]) + 4 + len(entry) + (4 if new else 0) + if b["count"] and size > BODY_LIMIT: + return False + if new: + b["modes"].append(new) + b["mtime"], b["count"] = mtime, b["count"] + 1 + b["entries"] += entry + return True + + for ordinal, value in rows: + value = bytes(value) + if ordinal <= 0 or len(value) != VALUE_BYTES or (previous is not None and ordinal <= previous): + raise ValueError("values must be ascending distinct 73-byte rows") + follows = previous is not None and ordinal == previous + 1 + previous = ordinal + if block and follows and push(block, value): + continue + if block: + finish(block) + block = {"first": ordinal, "count": 0, "modes": [], "mtime": 0, "entries": bytearray()} + push(block, value) + if block: + finish(block) + return blocks + + +def decode_values(first, last, body, fields=None): + reader = Reader(opened(body)) + if reader.byte() != 1: + raise ValueError("value block format") + count = reader.byte() + if count == 0 or last - first + 1 != count: + raise ValueError("value block count") + modes = [reader.take(4) for _ in range(reader.byte())] + fields = fields if fields is not None else Counter() + fields["header"] += 3 + 4 * len(modes) + 4 + mtime, values = 0, [] + for _ in range(count): + start = reader.at + tag = reader.byte() + if tag == RAW: + values.append(reader.take(VALUE_BYTES)) + fields["raw"] += reader.at - start + continue + at = reader.at + refs = reader.varint() + fields["refcount"] += reader.at - at + value = bytes([tag]) + refs.to_bytes(8, "big") + reader.take(32) + fields["content_root"] += 32 + if tag & 0x80 == 0: + value += reader.take(32) + fields["attribute_root"] += 32 + else: + value += modes[reader.byte()] + at = reader.at + mtime = wrap64(mtime + unzigzag(reader.varint())) + fields["mtime"] += reader.at - at + value += mtime.to_bytes(8, "big", signed=True) + reader.take(4) + bytes(16) + fields["mode_index+nanos"] += 5 + fields["tag"] += 1 + values.append(value) + if reader.at != len(reader.data): + raise ValueError("value block trailing bytes") + return values + + +def encode_hints(rows): + """[(grp, body)] of one write's (child, parent, name) hints, as the engine packs them.""" + hints = sorted({(c, p, bytes(n)) for c, p, n in rows}) + if any(c == 0 or p == 0 or not 1 <= len(n) <= 255 for c, p, n in hints): + raise ValueError("binding hint out of range") + blocks, block = [], None + + def finish(b): + body = bytes([1]) + varint(b["count"]) + bytes(b["entries"]) + blocks.append((b["group"], body + crc32c(body).to_bytes(4, "little"))) + + def push(b, hint): + child, parent, name = b["previous"] or ((b["group"] << GROUP_BITS), 0, b"") + common = 0 + while common < min(len(name), len(hint[2]), 255) and name[common] == hint[2][common]: + common += 1 + entry = varint(hint[0] - child) + varint(zigzag(hint[1] - parent)) + entry += bytes([common]) + varint(len(hint[2]) - common) + hint[2][common:] + size = 1 + len(varint(b["count"] + 1)) + len(b["entries"]) + len(entry) + 4 + if b["count"] and size > BODY_LIMIT: + return False + b["entries"] += entry + b["count"] += 1 + b["previous"] = hint + return True + + for hint in hints: + group = hint[0] >> GROUP_BITS + if block and block["group"] == group and push(block, hint): + continue + if block: + finish(block) + block = {"group": group, "count": 0, "entries": bytearray(), "previous": None} + push(block, hint) + if block: + finish(block) + return blocks + + +def decode_hints(group, body, fields=None): + reader = Reader(opened(body)) + if reader.byte() != 1: + raise ValueError("hint block format") + count = reader.varint() + fields = fields if fields is not None else Counter() + fields["header"] += reader.at + 4 + child, parent, name, hints = group << GROUP_BITS, 0, b"", [] + for _ in range(count): + at = reader.at + child += reader.varint() + parent = wrap64(parent + unzigzag(reader.varint())) % 2**64 + fields["child+parent"] += reader.at - at + common = reader.byte() + rest = reader.varint() + if common > len(name): + raise ValueError("hint name prefix") + name = name[:common] + reader.take(rest) + fields["name"] += 1 + len(varint(rest)) + rest + hint = (child, parent, name) + if child >> GROUP_BITS != group or (hints and hints[-1] >= hint): + raise ValueError("hint order") + hints.append(hint) + if reader.at != len(reader.data): + raise ValueError("hint block trailing bytes") + return hints + + +def attribute(values, hints): + """Summary of [(first, last, body)] value blocks and [(grp, seq, body)] hint blocks.""" + vf, hf = Counter(), Counter() + decoded = [v for first, last, body in values for v in decode_values(first, last, body, vf)] + forms = Counter( + "raw" if v[0] == RAW or (v[0] & 0x80 and any(v[57:])) else "inline" if v[0] & 0x80 else "root" for v in decoded + ) + entries = sum(len(decode_hints(g, body, hf)) for g, _, body in hints) + groups = defaultdict(int) + for g, _, _ in hints: + groups[g] += 1 + vb, hb = sum(len(b) for *_, b in values), sum(len(b) for *_, b in hints) + return { + "label": "DIAGNOSTIC", + "inode_value_block": { + "blocks": len(values), + "entries": len(decoded), + "body_bytes": vb, + "body_bytes_per_entry": round(vb / len(decoded), 2) if decoded else None, + "forms": dict(forms), + "duplicate_values": len(decoded) - len(set(decoded)), + "field_bytes": dict(vf), + }, + "binding_hint_block": { + "blocks": len(hints), + "groups": len(groups), + "max_blocks_per_group": max(groups.values(), default=0), + "entries": entries, + "body_bytes": hb, + "body_bytes_per_entry": round(hb / entries, 2) if entries else None, + "field_bytes": dict(hf), + }, + } + + +def _read_sql(*parts): + with open(os.path.join(SQL, *parts)) as stream: + return stream.read() + + +def _source(version, layout, tables, without=()): + """The schema source the engine records for a version (scripts in its order), less `without`.""" + objects = {16: "objects.sql", 17: "objects_units.sql", 18: "objects_units.sql"}[layout] + scripts = [_read_sql("sqlite", objects) + (_read_sql("sqlite", "objects_units_index.sql") if layout == 18 else "")] + scripts += [ + _read_sql("sqlite", n).replace("schema_version=1", f"schema_version={version}") + for n in ("metadata.sql", "history.sql") + ] + scripts += [_read_sql("sqlite", n) for n in ("lineage.sql", "merge_scratch.sql", "file_digest.sql")] + if tables: + scripts.append(_read_sql("sqlite", "acquisition", "schema.sql")) + source = "".join(scripts) + for text in without: + source = source.replace(text, "", 1) + return source + + +def _statement(script, head): + start = script.index(head) + return script[start : start + script[start:].index(";\n") + 1] + + +def _definition(con): + rows = con.execute("SELECT type,name,sql FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' ORDER BY type,name") + return "".join(f"{t}\n{n}\n{s}\n" for t, n, s in rows) + + +def migrate(source_path, target_path): + if os.path.exists(target_path) or os.path.exists(source_path + "-wal") and os.path.getsize(source_path + "-wal"): + raise SystemExit("refused: target exists, or the source is not closed (non-empty WAL)") + shutil.copyfile(source_path, target_path) + con = sqlite3.connect(target_path, isolation_level=None) + old = con.execute("PRAGMA user_version").fetchone()[0] + if con.execute("PRAGMA application_id").fetchone()[0] != APPLICATION_ID or old not in (16, 17, 18, 22, 23, 24): + raise SystemExit(f"refused: not a schema version 1 Store (user_version {old})") + tables, layout, new = old >= 22, old - (6 if old >= 22 else 0), old + 20 + recorded, definition = con.execute("SELECT schema_source,schema_definition FROM history_meta").fetchone() + if definition != _definition(con): + raise SystemExit("refused: live schema differs from the recorded definition") + metadata, history = _read_sql("sqlite", "metadata.sql"), _read_sql("sqlite", "history.sql") + v2_values, v2_hints = ( + _statement(metadata, "CREATE TABLE inode_value_block ("), + _statement(history, "CREATE TABLE binding_hint_block ("), + ) + # The open-time additive migrations add these, newest first; a Store may lack a suffix of them. + optional = [_read_sql("sqlite", n) for n in ("file_digest.sql", "merge_scratch.sql", "lineage.sql")] + hint_table = f"{v2_hints}\n\n" + for missing in range(len(optional) + 2): + without = optional[:missing] + ([hint_table] if missing == len(optional) + 1 else []) + v1 = _source(old, layout, tables, without).replace(v2_values, V1_VALUES).replace(v2_hints, V1_HINTS) + if recorded == v1: + break + else: + raise SystemExit("refused: the recorded schema is not one this release wrote") + con.execute("BEGIN IMMEDIATE") + values = encode_values(con.execute("SELECT ordinal,value FROM inode_value ORDER BY ordinal").fetchall()) + con.execute("DROP TABLE inode_value") + con.execute(v2_values) + con.executemany("INSERT INTO inode_value_block VALUES (?,?,?)", values) + has_hints = "binding_hint" in {r[0] for r in con.execute("SELECT name FROM sqlite_schema WHERE type='table'")} + if has_hints: + hints = encode_hints(con.execute("SELECT child,parent,name FROM binding_hint").fetchall()) + con.execute("DROP TABLE binding_hint") + con.execute(v2_hints) + seq = Counter() + for group, body in hints: + seq[group] += 1 + con.execute("INSERT INTO binding_hint_block VALUES (?,?,?)", (group, seq[group], body)) + con.execute("PRAGMA writable_schema=ON") + for table in ("store_policy", "history_meta"): + con.execute( + "UPDATE sqlite_schema SET sql=replace(sql,?,?) WHERE type='table' AND name=?", + (f"schema_version={old})", f"schema_version={new})", table), + ) + con.execute("PRAGMA writable_schema=OFF") + con.execute("COMMIT") + con.close() + con = sqlite3.connect(target_path, isolation_level=None) + con.execute("BEGIN IMMEDIATE") + con.execute(f"PRAGMA user_version={new}") + con.execute("UPDATE store_policy SET schema_version=?", (new,)) + source = _source(new, layout, tables, without) + con.execute( + "UPDATE history_meta SET schema_version=?,schema_source=?,schema_definition=?", (new, source, _definition(con)) + ) + con.execute("COMMIT") + if con.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise SystemExit("integrity check failed; remove the target") + con.execute("VACUUM") + con.execute("PRAGMA wal_checkpoint(TRUNCATE)") + con.close() + for suffix in ("-wal", "-shm"): + if os.path.exists(target_path + suffix) and not os.path.getsize(target_path + suffix): + os.remove(target_path + suffix) + return {"user_version": [old, new], "value_blocks": len(values), "hint_blocks": len(hints) if has_hints else 0} + + +def main(argv): + if len(argv) == 3 and argv[1] == "attribute" and argv[2] != "--rows": + con = sqlite3.connect(f"file:{argv[2]}?immutable=1", uri=True) + values = con.execute("SELECT first_ordinal,last_ordinal,body FROM inode_value_block ORDER BY 1").fetchall() + hints = con.execute("SELECT grp,seq,body FROM binding_hint_block ORDER BY 1,2").fetchall() + print(json.dumps(attribute(values, hints), indent=1)) + elif len(argv) == 4 and argv[1:3] == ["attribute", "--rows"]: + values, hints = [], [] + with open(argv[3]) as stream: + for line in stream: + kind, a, b, body = line.rstrip("\n").split("\t") + (values if kind == "value" else hints).append((int(a), int(b), bytes.fromhex(body))) + print(json.dumps(attribute(sorted(values), sorted(hints)), indent=1)) + elif len(argv) == 4 and argv[1] == "migrate": + print(json.dumps(migrate(argv[2], argv[3]))) + else: + raise SystemExit(__doc__) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/tools/test_store_blocks.py b/tools/test_store_blocks.py new file mode 100644 index 0000000000..a8f9f55230 --- /dev/null +++ b/tools/test_store_blocks.py @@ -0,0 +1,74 @@ +"""Tests for store_blocks.py: the engine's golden blocks (engine tests +sqlite_inode_values.rs and binding_hint_blocks.rs assert the same bytes), +round trips of every value shape, and checksum refusal.""" + +import random +import unittest + +import store_blocks as sb + +GOLDEN_VALUES = bytes.fromhex( + "010402000081a4000081ed810111111111111111111111111111111111111111111111111111111111111111110080c49fd50c0102030481011111111111111111111111111111111111111111111111111111111111111111010a01020304010222222222222222222222222222222222222222222222222222222222222222223333333333333333333333333333333333333333333333333333333333333333fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff2f3427f" +) +GOLDEN_HINTS = [ + (0, bytes.fromhex("010307020005612e74787400000005622e74787402020005612e7478742d1e97ab")), + (1, bytes.fromhex("0101060200017860fbeefb")), +] + + +def inline(mode, mtime, refs=1): + head = bytes([0x81]) + refs.to_bytes(8, "big") + bytes([0x11]) * 32 + mode.to_bytes(4, "big") + return head + mtime.to_bytes(8, "big", signed=True) + bytes([1, 2, 3, 4]) + bytes(16) + + +GOLDEN_ROWS = [ + inline(0o100644, 1_700_000_000), + inline(0o100755, 1_700_000_005), + bytes([1]) + (2).to_bytes(8, "big") + bytes([0x22]) * 32 + bytes([0x33]) * 32, + bytes([0xFF]) * 73, +] +HINTS = [(7, 1, b"a.txt"), (7, 1, b"b.txt"), (9, 2, b"a.txt"), (70, 1, b"x")] + + +class StoreBlocksTest(unittest.TestCase): + def test_golden_value_block(self): + self.assertEqual(sb.decode_values(5, 8, GOLDEN_VALUES), GOLDEN_ROWS) + self.assertEqual(sb.encode_values(list(zip(range(5, 9), GOLDEN_ROWS))), [(5, 8, GOLDEN_VALUES)]) + + def test_golden_hint_blocks(self): + self.assertEqual(sb.encode_hints(HINTS + HINTS[:1]), GOLDEN_HINTS) + decoded = [h for g, body in GOLDEN_HINTS for h in sb.decode_hints(g, body)] + self.assertEqual(decoded, HINTS) + + def test_every_value_shape_round_trips(self): + rng = random.Random(91) + rows = [] + for ordinal in range(1, 2000): + value = bytearray(rng.randbytes(73)) + shape = ordinal % 4 + if shape == 0: + value[0] = 0x80 | rng.randrange(1, 0x7F) + value[57:] = bytes(16) + elif shape == 1: + value[0] &= 0x7F + if ordinal % 97 == 0: + continue # a gap starts a new block + rows.append((ordinal, bytes(value))) + blocks = sb.encode_values(rows) + self.assertTrue(all(len(body) <= sb.BODY_LIMIT for _, _, body in blocks)) + decoded = [ + (first + i, v) for first, last, body in blocks for i, v in enumerate(sb.decode_values(first, last, body)) + ] + self.assertEqual(decoded, rows) + + def test_a_changed_byte_fails_the_checksum(self): + body = bytearray(GOLDEN_VALUES) + body[10] ^= 1 + with self.assertRaises(ValueError): + sb.decode_values(5, 8, bytes(body)) + with self.assertRaises(ValueError): + sb.decode_values(5, 9, GOLDEN_VALUES) + + +if __name__ == "__main__": + unittest.main()