diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 2d36f4d..0a99d57 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## 项目定位 -slotsboxmalloc 是基于 16 叉伙伴系统（16-ary buddy system）的存储分配器。不是通用 malloc 替代，设计目标覆盖 OS kernel 和 block 设备的存储分配场景。依赖 blockmalloc 管理 meta 区的 box_head_t 节点。 +slotsboxmalloc 是基于 N=64 伙伴树（left-box/right-obj）的存储分配器。不是通用 malloc 替代，设计目标覆盖 OS kernel 和 block 设备的存储分配场景。全部实现都在头文件 `include/slotsboxmalloc/slotsboxobj.h`（header-only，`src/slotsboxmalloc.c` 只是实例化），依赖 blockmalloc 管理 meta 区的 box 节点。性能优化细节与实验数据见 `doc/perf-v3.md`。 ## 构建 @@ -12,232 +12,65 @@ slotsboxmalloc 是基于 16 叉伙伴系统（16-ary buddy system）的存储分 make ``` -**注意**：slotsboxmalloc 依赖 blockmalloc 共享库。需要先 `sudo make install` blockmalloc，或设置 `LD_LIBRARY_PATH`。运行时确保 `libblockmalloc.so.1` 在链接路径中。 +**注意**：slotsboxmalloc 依赖 blockmalloc 共享库。需要先 `sudo make install` blockmalloc，或设置 `LD_LIBRARY_PATH`。运行时确保 `libblockmalloc.so.1` 在链接路径中。无 cmake 时可手动： + +```bash +gcc -O3 -DNDEBUG -fPIC -shared -std=c11 -Iinclude -I/include \ + src/slotsboxmalloc.c -L/build -lblockmalloc -o libslotsboxmalloc.so +``` ## 测试 ```bash -# Python test runner（含耗时、exit code、CSV 报告，默认并行数=cpu_count） python3 tutorial/test.py python3 tutorial/test.py --filter 04 # 按名称过滤 -python3 tutorial/test.py --repeat 10 # 每 case 重复 N 轮 python3 tutorial/test.py --skip-stress # 跳过 03_stress（有无界循环） -python3 tutorial/test.py --jobs 1 # 单线程串行（调试时用） -python3 tutorial/test.py --errorexit # 遇错即停 - -# 直接运行单个 case -cd build && ./tutorial/01_basic - -# Sanitizer 下运行 +cd build && ./tutorial/01_basic # 直接运行单个 case ASAN_OPTIONS=detect_leaks=1 ./tutorial/01_basic -TSAN_OPTIONS=history_size=7 ./tutorial/04_multithread ``` -tutorial case 清单： +tutorial case 清单（节选）： | case | 说明 | |------|------| | `01_basic` | 基础 init/alloc/free | -| `02_bench` | 111 次不同大小 alloc/write/read/free | -| `03_stress` | 分配至满，然后无限随机 free/alloc 循环 | -| `04_multithread` | 多线程并发 alloc/free（**当前 FAIL：slotsboxmalloc 无锁**）| -| `05_multiprocess` | 多进程并发，共享内存 mmap（间歇 FAIL）| -| `06_multiprocess_multithread` | 多进程 × 多线程（间歇 FAIL）| - -## 调试 - -```bash -# Valgrind 内存检查 -valgrind --leak-check=full --track-origins=yes ./tutorial/01_basic - -# GDB 断点调试 -gdb --args ./tutorial/01_basic - -# TSan 抑制假阳性（如必要的 benign race） -TSAN_OPTIONS="suppressions=tsan_suppress.txt history_size=7" ./tutorial/04_multithread -``` - ---- - -## 架构 - -### 16 叉伙伴树模型 - -slotsboxmalloc 将数据区视为一棵 16 叉树。最小分配单元 = 8 字节，按 16 的幂对齐： - -``` -16^0 * 8 = 8B 一个 slot -16^1 * 8 = 128B 16 个 slot -16^2 * 8 = 2KB 256 个 slot -... -16^N * 8 16^N 个 slot -``` - -- **level**: 树的层级，0 = 最小级别（8B/slot） -- **multiple**: 该层级占用几个连续 slot（1-15，或者占满 16 等价于 level+1, multiple=1） - -### 内存布局（两块独立区域） - -``` -meta 区（连续）: data 区（独立）: -┌─────────────────────┐ ┌─────────────────┐ -│ box_meta_t │ │ obj 数据 │ -│ magic[16] │ │ （纯数据, │ -│ boxhead_bytessize │ │ 无任何元信息) │ -│ box_bytessize │ │ │ -│ blocks_meta_t │ ← blockmalloc │ │ -├─────────────────────┤ 管理这部分 │ │ -│ box_head_t[0] (root) │ ← block_id 0 │ │ -│ box_head_t[1] │ │ │ -│ ... │ │ │ -└─────────────────────┘ └─────────────────┘ -``` - -- `metaptr` 指向 meta 区开头（即 `&box_meta_t`） -- `boxhead = metaptr + sizeof(box_meta_t)` 是 blockmalloc 管理的 block 区域 -- data 区的地址 slotsboxmalloc 不持有——调用者自行管理，box_alloc 返回的是 data 区内的**字节偏移量** - -### obj_usage：size 的内部编码 - -```c -typedef struct { - uint8_t level : 4; // 0-15, 16^level * 8 字节 - uint8_t multiple : 4; // 1-15, 连续 slot 数。multiple==0 表示无可用空间 -} obj_usage; -``` - -关键转换函数： -- `align_to(n)`: 将 8 字节单位数 n 转换为 (level, multiple) 对 -- `obj_offset(usage)`: 计算 `8 * 16^level * multiple` 字节偏移 -- `compare_obj_usage(a, b)`: 先比 level，再比 multiple - -**易错点**：`multiple == 0` 是哨兵值（表示 "无可用空间"），`multiple == 16` 等价于 `(level+1, multiple=1)`。 - -### box_head_t：树节点结构 - -```c -typedef struct { - uint8_t state : 2; // BOX_UNUSED/BOX_FORMATTED - int8_t max_obj_capacity : 6; // 当前节点最大连续空闲 slot 数 [0-16] - - atomic_int_fast64_t rw_lock; // 预留锁字段（★ 未使用） - - int32_t parent; // parent 的 block_id（-1=root） - uint8_t objlevel; // 当前节点所在 level - uint8_t avliable_slot; // 节点可用 slot 数 [2-16] - obj_usage child_max_obj_capacity; // 所有子节点中最大可用容量（hint） - - box_child_t used_slots[16]; // 每个 slot 的状态 - int32_t childs_blockid[16]; // 每个 slot 对应的子节点 block_id（-1=无子节点） -} box_head_t; -``` - -### Slot 状态机 - -``` -BOX_UNUSED (0) ──alloc obj──→ OBJ_START (2) + OBJ_CONTINUED (3)... - （obj 占 1+ 个连续 slot，首 slot=START，后续=CONTINUED） - -BOX_UNUSED (0) ──create child─→ BOX_FORMATTED (1) - （此 slot 指向一个子 box_head_t 节点） -``` - -- `OBJ_START` 和 `OBJ_CONTINUED` 的 `continue_max` 字段无意义（设为 0） -- `BOX_UNUSED` 的 `continue_max` 是一个 hint，表示从该位置开始的连续空闲 slot 数 - -### offset ↔ 树位置映射 - -这是最容易出错的环节。`box_alloc` 返回的 offset 是 data 区内的**字节偏移**。 - -**offset → node + slot_index**（`find_obj_node`）： -``` -unit_offset = obj_offset / 8; // 转换为 8 字节单位 -从 root 开始, current_level = root->objlevel: - slot_index = (unit_offset / 16^current_level) % 16; - 读 used_slots[slot_index]: - OBJ_START → 找到了，返回此 node + slot_index - BOX_FORMATTED → 进入子节点, current_level-- - 其他 → 错误 -``` - -**node + slot_index → offset**（用于分配时返回给调用者）： -``` -offset = obj_offset({level=node->objlevel, multiple=slot_index}) - = 8 * 16^level * slot_index -``` - -此 offset 是相对于 data 区起始的偏移。调用者在 data 区写入：`data + offset`。 - -### 核心算法 - -#### box_alloc - -``` -1. size → 8 字节对齐向上取整 → align_to → obj_usage (level, multiple) -2. 检查 root 的最大容量（child_max_obj_capacity hint），不够则失败 -3. box_find_alloc 递归遍历树： - a. objsize.level == node->objlevel → put_slots 分配连续 slot - b. objsize.level < node->objlevel → 遍历子节点: - - 已有子节点 (BOX_FORMATTED): 检查容量 hint，不够则跳过 - - 空闲 slot (BOX_UNUSED): 创建子节点（blocks_alloc），递归进入 -4. 返回 offset（可能是多层累加：parent_offset + child_offset） -``` - -**关键：多层 offset 累加**。当分配在子节点中完成时，返回的 offset = 当前层 slot 偏移 + 子节点内偏移。每个子节点覆盖父节点对应 slot 的地址空间（1/16）。 - -#### box_free - -``` -1. find_obj_node: offset → node + slot_index -2. 释放 slot(s): OBJ_START → BOX_UNUSED, OBJ_CONTINUED → BOX_UNUSED -3. 重新计算 max_obj_capacity（box_continuous_max） -4. 如果 max_obj_capacity 变化且 parent 存在 → update_parent 递归上传 -``` - -#### update_parent - -从子节点向上传播容量变化： -- `slotstate_changed`: 本节点 max_obj_capacity 变了 → 更新 parent 的 child_max_obj_capacity -- `slot_max_obj_capacity_changed`: 子节点的 child_max 变了 → 当前节点重新扫描所有子节点取 max → 继续上传 -- 递归直到 root 或变化消失 - -**注意**：`update_parent` 涉及 `child_max_obj_capacity` 的重新计算——遍历所有 `BOX_FORMATTED` 的 slot，读子节点的 `box_and_child_max_obj_capacity()` 取 max。 - -#### box_continuous_max +| `02_bench` | alloc/write/read/free 计时 | +| `03_stress` | 分配至满（2MB 数据区），然后随机 free/alloc 循环（2s 限时）| +| `04_multithread` | 多线程并发 alloc/free | +| `05_multiprocess` / `06_multiprocess_multithread` | 共享内存多进程并发 | +| `08_bench_pure` / `16_bench_malloc` | 纯吞吐基准（16 含 glibc 对照）| +| `17_data_integrity` | 数据完整性 | +| `18_model_check` | 随机模型校验：不重叠/容量精确/多线程零假失败（ctest）| -扫描节点所有 slot，找最长连续 `BOX_UNUSED` 段。时间复杂度 O(avliable_slot) ≤ 16。 +## 架构 (v3) ---- +### 64 叉树模型 -## 并发 +数据区必须为 `8 × 64^k` 字节。最小分配单元 8 字节：L1=512B（64×8B）, L2=32KB, L3=2MB, L4=128MB…每 box 固定 64 个 slot，子 box 从左侧分配、对象从右侧分配（连续多 slot，首 slot 状态字节内嵌长度）。 -### 当前状态：无锁 +- **level / multiple**：`sbo_align_to(n)` 把 8 字节单位数换成 `(level, multiple)`，multiple ∈ [1,63]；容量编码 `enc=(level<<6)|multiple` 可直接整数比较。 +- **free_mask**：单个 `uint64_t`，bit=1 表示 slot 空闲。单段连续空闲走 `clz`/`ctz` O(1) 路径，碎片状态走 SWAR 词运算。 -`lock.h` 定义了 `rlock`/`runlock`/`lock`/`unlock` 函数，`box_head_t.rw_lock` 字段也存在，但**代码中完全没有调用**。注释中的 "线程安全需求" 是目标设计，尚未实现。 +### 内存布局 -### 并发测试结果 +meta 区与 data 区分离。`sbo_alloc` 返回 data 区内的**字节偏移**，调用者自行 `data + offset`。meta 布局：`[sbo_meta_t][N×blocks_meta_t][N×sbo_lock_t][shard0 池][shard1 池]...`，池内每个块 = blockmalloc 块头 + 对齐 pad + `sbo_box_t`（恒 8 字节对齐，步长为 8 的倍数）。 -- 04_multithread: 稳定 FAIL，alloc 很快失败（树状态被并发破坏） -- 05_multiprocess: 间歇 FAIL（进程间竞态） -- 06_multiprocess_multithread: 间歇 FAIL +### 分片与并发 -### 修复计划 +- 数据区 ≥ 64×2MB 时切成 64 个 shard（`SBO_MIN_SHARD_BYTES` 可调），每 shard 独立锁、独立 blockmalloc 池、独立树根；否则单 shard。单次分配上限 ≈ shard 大小的 63/64。 +- `sbo_alloc`：TLS 亲和 home shard，先 trylock 轮转全部 shard、再阻塞轮转，**不会因锁竞争误报 OOM**。`sbo_free` 由 offset 移位定位 shard 后加锁。 +- 容量 hint 与直达缓存：父 box 的 `child_cap[64]` 缓存各子 box 的 combined 容量，`child_max_enc` 用持有者计数 + `child_mask` 定向重扫维护，变化沿父链 O(1) 传播；每 shard 锁内缓存 L1/L2/L3 活跃 box ID 与基址，命中时 O(1) 直达叶节点。 +- 锁是 per-shard TTAS spinlock（acquire/release），位于 meta 区内，SHM 多进程可用。box 内无锁，同一 shard 的所有操作由 shard 锁串行化。 -根节点自旋锁（最小改动方案）： -1. `box_head_t` 已有 `rw_lock` 字段，不需要改数据结构 -2. `lock.h` 已有 spinlock 函数（需要从 rwlock 简化为普通 spinlock） -3. `box_alloc` 和 `box_free` 入口处加 `spin_lock(&root->rw_lock)` / `spin_unlock` -4. 临界区极短（16 叉树深 ~7 层，50-100 cycles），一把锁完全够用 +### offset ↔ 树位置 ---- +从 shard 根起按 6-bit 逐层解码 unit offset（`sbo_find_obj_node`）：每层取 `(unit >> 6*(level-1)) & 63`，`child_mask` 命中则下降、`OBJ_START` 且低位对齐则命中。shard 边界与高位 6-bit 对齐，因此同一解码对分片/未分片都成立。 ## 常见易错点 -1. **offset 是相对 data 区的字节偏移，不是绝对地址**。调用者自己 `data + offset`。 -2. **`multiple == 0` 是哨兵**（无可用空间），`multiple == 16` 应进位到 `(level+1, 1)`。 -3. **`box_continuous_max` 只在 `BOX_UNUSED` 上计数**。OBJ_START/CONTINUED 和 BOX_FORMATTED 都会打断连续计数。 -4. **`box_and_child_max_obj_capacity` 取自身容量和子树容量的 max**，两者可能不同（一个满但有子节点空间，或全空但无子节点）。 -5. **`childs_blockid[i]` 不是指针，是 blockmalloc 的 block_id**。通过 `boxhead + blockdata_offset(&meta->blocks, id)` 获取实际指针。 -6. **`avliable_slot` 不是恒定 16**。虽然新格式化的节点初始化为 16，但理论上可以更小（预留字段）。 -7. **`__attribute__((packed))` 在多处使用**：`box_child_t`、`box_head_t`、`obj_usage`。修改这些结构体时必须保持 packed 且检查 sizeof。 -8. **位域宽度精确匹配**：`obj_usage.level:4` 最大 15，`box_child_t.continue_max:6` 最大 63（但语义上是 0-16）。修改位域宽度时检查所有使用点的取值范围。 +1. **offset 是相对 data 区的字节偏移**，不是指针；meta 格式为 v3（`slotsboxobj64v3`），与 v2 不兼容，旧 meta 必须重新 init。 +2. **空子 box 按需惰性回收**：常规 free 保持空子 box 温热复用；当父层空间不足或 meta 块耗尽时由 `sbo_reclaim_empty` 自动回收全空子树。 +3. **meta 太小会自动退回单 shard**（每 shard 池 < 32 个 box 时），多线程扩展性随之消失——压测时先确认 `sbo_meta_size` 给足。 +4. **`sbo_box_t` 不是 packed**，靠 init 时的块填充保证对齐；改结构体后检查 `block_stride`/`box_off` 仍为 8 的倍数（UBSan alignment 会报）。 +5. **shard 根恒为 block 0**，运行时据此 O(1) 定位；不要在 init 里先于根分配其它块。 +6. `multiple==64` 不存在：恒进位为 `(level+1, 1)`，单 box 自身 `max_obj_cap` 上限为 63（`SBO_MULT_MAX`）；`enc==0` 表示无容量。 diff --git a/doc/perf-v3.md b/doc/perf-v3.md new file mode 100644 index 0000000..01f4d43 --- /dev/null +++ b/doc/perf-v3.md @@ -0,0 +1,96 @@ +# 性能优化报告 (v3) + +对象: `include/slotsboxmalloc/slotsboxobj.h`（唯一的实现文件）。 +环境: Intel Xeon w7-3455 (48 核), gcc 13.2, `-O3 -DNDEBUG`（与 CMake Release 相同）, +经共享库链接调用。数据区 128MB (`8*64^4`), meta 16MB。 +所有数字为 5 次运行中位数; 吞吐按**成功**操作计数, 失败率单独列出。 + +## 基线 (v2) 的四个问题 + +1. **多线程假失败**: 128MB 数据区在 v2 只有一个 root slot（一把锁）, 且 + `sbo_alloc` 用 trylock, 抢不到锁直接返回 -1。8 线程时 94.5% 的分配 + 在空间充足时失败（仓库自带 bench 只数成功数, 掩盖了这一点）。 +2. **填充二次退化**: alloc-only 保持存活的场景下, 每次分配从下标 0 + 线性扫描子 box 并逐个解引用, 50k 次分配平均 47µs/op 且随填充恶化。 +3. **逐位扫描**: free bitmap 是 `uint8_t[8]`, 连续最大空闲、找连续 run、 + 置/清位全是逐 bit 循环; `update_parent` 每次容量变化都重扫全部子 + box 并解引用各自指针; `packed` 结构体使每次成员访问都是不对齐访问。 +4. **潜在越界**: `sbo_alloc_obj_slots` 在 `start==0` 时 + `obj_boundary = start-1` 下溢为 255, 随后 `sbo_alloc_box_slot` 会 + 扫描并写入 slot 64 以外的内存（基线在压测中复现过段错误, 8 线程 + 模型检查出现过 `allocated_size=0`）。 + +## v3 的改动 + +- `free_mask` 改为单个 `uint64_t`: 单段连续空闲走 `clz`/`ctz` O(1) 快速路径, + 碎片状态走 6 步 SWAR 倍增位移与（无分支循环）计算连续最大与连续 run。 +- slot 状态改为一字节（紧邻 32B 热头布局在 `sbo_box_t` 前 96B 内）, + `OBJ_START` 内嵌对象长度: free 与 `allocated_size` O(1) 取长, 校验低位对齐防止内部偏移误判。 +- 父 box 缓存每个子 box 的 combined 容量（uint16 编码 `(level<<6)|mult`, + `mult` 截断至 `SBO_MULT_MAX=63` 消除 `64<->63` 传播风暴与跨层虚报）, + 下降时先校验 `child_max_enc >= need_enc` 与 `child_cap[i]`；最大值用 + “持有者计数 + `child_mask` 定向重扫”精确维护, 父节点 combined 不变时立即停止传播。 +- 每 shard 锁结构（64B cache line 填充区）内嵌 L1/L2/L3 活跃 box 直达索引 + (`hot_id1`/`hot_base`), `sbo_alloc` 与 `sbo_free` 命中热 box 时 O(1) 直达叶节点, + 无需从 root 逐层下探; 配合 `hint_child` roving 提示保证填充场景 O(depth)。 +- 按需惰性回收空子 box（`sbo_reclaim_empty`）与深层 `blocks_alloc` 失败回滚: + 常规 churn 保持叶 box 温热零抖动, 当父层满或 meta 块耗尽时自动回收空子树。 +- **分片**: 数据区 ≥ 64×2MB 时切为 64 个独立 shard（各自 TTAS 锁 + blockmalloc + 池 + 树根）, 线程用 `initial-exec` TLS 亲和（奇数步长 9 错开相邻线程 cache line）; + `sbo_alloc` 先 trylock 轮转、再阻塞轮转, 瞬时竞争不盲目迁移 home shard, + 不再因竞争误报 OOM。 +- box 经填充保证恒 8 字节对齐（块头 2/4 字节时加 pad, 步长取 8 的倍数）, + 去掉 `packed` 的不对齐访问与冗余 `memset`；ASan/UBSan/TSan 干净。 + +代价/兼容性: + +- meta 格式不兼容 v2（magic 升级为 `slotsboxobj64v3`）, 需重新 `sbo_init`。 +- 单个 box 元数据 343B → 480B（多了 child_cap 缓存等）, 相同 meta 区可放 + 的 box 数约减少 29%。 +- 单次分配上限约为 shard 大小的 63/64（128MB 数据区 → 约 1.97MB）, + 这是 v2“根 box 63 个 slot”同一规则在分片后的形式；可用 + `SBO_MIN_SHARD_BYTES` 调大 shard 下限换取更大单次上限（分片数随之减少）。 +- 极小数据区（<512B）现在 `sbo_init` 直接拒绝（v2 接受但根 level 为 0 + 实际不可用）；meta 区要求 8 字节对齐。 + +## 结果（自研严格基准, 成功吞吐, 中位数） + +alloc+free 混合尺寸 {8,64,512,4096}B, 每线程 200k 次（`taskset -c 0-15`）: + +| 线程 | v2 Mops/s | v2 失败率 | v3 (initial) Mops/s | v3 (optimized) Mops/s | 相比 v2 提升 | +|---:|---:|---:|---:|---:|---:| +| 1 | 3.27 | 0% | 16.93 | **20.33** | **6.2×** | +| 4 | 1.62 | 91.7% | 64.09 | **80.31** | **49.6×** | +| 8 | 0.85 | 95.0% | 119.53 | **162.55** | **191×** | +| 16 | 0.75 | 95.0% | 180.64 | **248.46** | **331×** | + +其它场景（1 线程 / 16 线程）: + +- 小尺寸 {8..64}B alloc+free: 3.11 → **20.42 Mops/s**（6.6×）/ 0.75 → **227.36 Mops/s**（303×） +- 随机 churn（2048 存活集, 8B–64KB）: 2.73 → **13.06 Mops/s**（4.8×）/ 0.83 → **112.74 Mops/s**（136×） +- alloc-only 填充 50k 不释放: 0.02 → **15.22 Mops/s**（761×, 消除二次退化） +- v3 在所有多线程场景失败率 0（空间充足时）。 + +仓库自带基准: + +- `08_bench_pure`: 单线程 **23.5M ops/s**; 8 线程全部完成 400000 次, **141.7M ops/s**。 +- `15_bench_dedicated`: 8 线程 **159.5M ops/s**。 + +## 正确性验证 + +- `tutorial/18_model_check.c`（ctest 以 `4 10000` 运行）: 随机 churn + 下用逐 8B 单位的 claim 表验证分配永不重叠、首尾图案验证存活对象不被 + disturb、`sbo_allocated_size` 恒等于模型容量, 且多线程空间充足时零假失败; + 另含溢出尺寸、shard 上限、L1/L2 内部偏移与未对齐/双重 free、过小 `head_size`、 + meta 块耗尽回滚、空子 box 回收合并、错误 magic、小数据区与单 shard 多线程检查。 +- `ctest` 10/10 与 `tutorial/test.py` 17/17 全部通过; + ASan+UBSan 与 TSan（`-fno-sanitize-recover=all`）全部干净。 + +## 复现 + +```bash +make test +./build/tutorial/08_bench_pure 8 +./build/tutorial/16_bench_malloc +./build/tutorial/18_model_check 16 100000 +``` diff --git a/include/slotsboxmalloc/doc.h b/include/slotsboxmalloc/doc.h index 165d7c8..6b79ca9 100644 --- a/include/slotsboxmalloc/doc.h +++ b/include/slotsboxmalloc/doc.h @@ -1,348 +1,6 @@ -/* - * doc.h — N-slots-box-obj 模型伪代码 - * - * 严格遵循 left-box / right-obj 布局以避免碎片化。 - * N 可配置 (#define SBO_N 64)，以下用 N 代指。 - * - * physical slot 布局（单 box 内）: - * ┌──────────────┬────────────────────────────┬──────────────────┐ - * │ 子 box 区 │ 空闲区 │ 对象区 │ - * │ [0, box_b) │ [box_b, obj_b] │ (obj_b, N-1] │ - * │ box→ 向右扩展 │ 连续空闲，可合并 │ obj← 向左扩展 │ - * └──────────────┴────────────────────────────┴──────────────────┘ - * - * - 分配子 box: 取 slot[box_boundary], box_boundary++ - * - 分配对象: 取 slot[obj_boundary-m+1 .. obj_boundary], obj_boundary -= m - * - 释放: 仅标记 FREE，不调边界。N=64 统计吸收碎片，不做 compact。 - * - * 约定: physical_slot ∈ [0,N-1]。无 compact → physical == virtual 始终成立。 - * 外部 offset 直接编码 physical slot index。 - * - * 模型: L0=8B(无slot), L_k = N^k × 8B (k≥1), 每 box 固定 N 个 slot. - * N_BITS = log2(N), offset 每层占 N_BITS bit. - * - * N=64 时各层 data 大小: - * L0 = 8B (原子, 无 slot) - * L1 = 512B (64 个obj，每个 8B) - * L2 = 32KB () - * L3 = 2MB () - * L4 = 128MB (对普通小程序，足够) - * L5 = 8GB - * L6 = 512GB - * 低层数带来的好处:malloc时， - */ - #ifndef SBO_DOC_H #define SBO_DOC_H -/* - * 假设 #define: - * SBO_N = 64 // 每 box 的 slot 数 - * SBO_N_BITS = 6 // log2(SBO_N) - * SBO_N_MASK = 0x3F // SBO_N - 1 - * SBO_BITMAP_B = SBO_N/8 // free_bitmap 字节数 - */ - -/* ================================================================ - * 1. 数据结构 - * ================================================================ */ - -/* - * sbo_usage_t (2 bytes) - * 容量编码: 8 × N^(level-1) × multiple bytes (level≥1) - * level ∈ [1,15], multiple ∈ [1,N-1], level=0 为 sentinel - */ - -/* - * sbo_slot_t (1 byte) - * state:2 — FREE(0) | BOX(1) | OBJ_START(2) | OBJ_CONT(3) - * _rsv:6 - */ - -/* - * sbo_box_t (每个 box 固定 N 个 slot) - * - * // --- 身份 --- - * uint8_t state; // BOX (formatted) - * uint8_t objlevel; // 本 box 所在 level (≥1) - * uint8_t slot_id; // 所属 root slot - * int32_t parent; // 父节点 block id (0=root, -1=无) - * - * // --- 物理布局 --- - * uint8_t free_bitmap[SBO_BITMAP_B]; // bit=1→FREE, bit=0→占用 - * sbo_slot_t slots[N]; // 按物理 slot 索引 - * int32_t children[N]; // 子 box 的 block id (-1=无) - * - * // --- left-box / right-obj 边界 --- - * uint8_t box_boundary; // boxes ∈ [0, box_boundary) - * uint8_t obj_boundary; // objs ∈ (obj_boundary, N-1] - * // free ∈ [box_boundary, obj_boundary] - * - * // --- 恒等映射 (无 compact, physical==virtual) --- - - * // --- 容量 hint --- - * sbo_usage_t child_max_cap; // 子树最大可用容量 (快速剪枝) - * uint8_t max_obj_cap; // 本 box 最大连续空闲 slot 数 [0,N] - */ - -/* - * sbo_meta_t - * magic[16], head_size, data_size - * slot_pools[MAX_ROOT_SLOTS] — per-slot blockmalloc - * slot_locks[MAX_ROOT_SLOTS] — per-slot spinlock (64B padding) - * slot_bytes, per_slot_meta, root_slots - * block_stride, sizeof_block_head - */ - - -/* ================================================================ - * 2. free_bitmap 辅助操作 (byte-oriented) - * ================================================================ */ - -/* - * 由于 free_bitmap 是 uint8_t[N/8]，所有位操作通过字节索引: - * - * bitmap_get(fb, i): - * return (fb[i >> 3] >> (i & 7)) & 1 - * - * bitmap_set(fb, i): - * fb[i >> 3] |= (1 << (i & 7)) - * - * bitmap_clear(fb, i): - * fb[i >> 3] &= ~(1 << (i & 7)) - * - * bitmap_test_mask(fb, start, count): - * // 检查 [start, start+count) 是否全为 1 - * for i in start .. start+count-1: - * if !bitmap_get(fb, i): return false - * return true - * - * bitmap_fill_all(fb): - * memset(fb, 0xFF, SBO_BITMAP_B) - * - * bitmap_clear_range(fb, start, count): - * for i in start .. start+count-1: - * bitmap_clear(fb, i) - * - * bitmap_set_range(fb, start, count): - * for i in start .. start+count-1: - * bitmap_set(fb, i) - * - * 优化: 若 SBO_N 是 64 的倍数，可将 8 字节 reinterpret 为 uint64_t 做批量操作。 - */ - - -/* ================================================================ - * 3. 核心操作伪代码 - * ================================================================ */ - -/* --- 3.1 box_format: 初始化一个 box --- - * - * sbo_box_format(box, objlevel, parent_id): - * box.state = BOX - * box.objlevel = objlevel - * box.parent = parent_id - * bitmap_fill_all(box.free_bitmap) // N 个 1 - * - * box.box_boundary = 0 - * box.obj_boundary = N - 1 // free 区 = [0, N-1] 全范围 - * - * for i in 0..N-1: - * box.slots[i].state = FREE - * box.children[i] = -1 - * box.v2p[i] = box.p2v[i] = i // 恒等映射 (预留) - * - * box.child_max_cap = {level: objlevel+1, mult: 1} // 满 box - */ - -/* --- 3.2 alloc_box_slot: 分配一个子 box slot（从左侧） --- - * - * sbo_alloc_box_slot(meta, box): - * for p = box.box_boundary to box.obj_boundary: - * if bitmap_get(box.free_bitmap, p): - * bitmap_clear(box.free_bitmap, p) - * box.slots[p].state = BOX - * box.box_boundary = p + 1 - * return (OK, slot=p) - * return FAIL // 空闲区无 slot → 向上传播失败 - */ - -/* --- 3.3 alloc_obj_slots: 分配 m 个连续对象 slot（从右侧） --- - * - * sbo_alloc_obj_slots(meta, box, m): // m ∈ [1, N-1] - * for p = box.obj_boundary down to box.box_boundary: - * start = p - m + 1 - * if start < box.box_boundary: break - * if bitmap_test_mask(box.free_bitmap, start, m): - * bitmap_clear_range(box.free_bitmap, start, m) - * box.slots[start].state = OBJ_START - * for j = start+1 to p: - * box.slots[j].state = OBJ_CONT - * box.obj_boundary = start - 1 - * return (OK, slot=start) - * return FAIL // 边界无连续空闲 → 向上传播失败 - */ - -/* --- 3.4 free_obj_slots: 释放对象的连续 slot --- - * - * sbo_free_obj_slots(meta, box, physical_start): - * p = physical_start - * assert box.slots[p].state == OBJ_START - * box.slots[p].state = FREE - * bitmap_set(box.free_bitmap, p) - * p++ - * while p < N and box.slots[p].state == OBJ_CONT: - * box.slots[p].state = FREE - * bitmap_set(box.free_bitmap, p) - * p++ - * // 不调边界 — 碎片由后续 alloc 扫描吸收 - */ - -/* --- 3.5 free_box_slot: 释放子 box slot --- - * - * sbo_free_box_slot(meta, box, physical_slot): - * assert box.slots[physical_slot].state == BOX - * box.slots[physical_slot].state = FREE - * bitmap_set(box.free_bitmap, physical_slot) - * box.children[physical_slot] = -1 - * // 不调边界, 不回收 child box_head_t (被动分配器) - */ - -/* --- 3.6 sbo_find_alloc: 递归树下降分配 --- - * - * sbo_find_alloc(meta, box, parent, usage): - * if usage.level == box.objlevel: - * r = sbo_alloc_obj_slots(meta, box, usage.multiple) - * if r == FAIL: return FAIL - * return sbo_offset_raw(box.objlevel, r.start_physical) - * - * if usage.level < box.objlevel: - * for p in 0..N-1: - * if children[p] >= 0: - * ch = sbo_child(meta, box, children[p]) - * if ch 子树容量 < usage: continue - * sub = sbo_find_alloc(meta, ch, box, usage) - * if sub != FAIL: - * return sbo_offset_raw(box.objlevel, p) + sub - * continue - * - * if slots[p] == FREE: - * r = sbo_alloc_box_slot(meta, box) - * if r == FAIL: continue - * child_id = blocks_alloc(...) - * box.children[r.physical] = child_id - * ch = sbo_child(meta, box, child_id) - * sbo_box_format(ch, box.objlevel-1, sbo_self_id(meta, box)) - * ch.slot_id = box.slot_id - * sub = sbo_find_alloc(meta, ch, box, usage) - * if sub != FAIL: - * return sbo_offset_raw(box.objlevel, r.physical) + sub - * return FAIL - */ - -/* --- 3.7 sbo_find_obj_node: offset → (box, slot) --- - * - * offset 分解: 每层占 N_BITS bit（无 compact → physical==virtual）。 - * unit = obj_offset / 8 - * 第 L 层的 slot = (unit >> ((L-1)*N_BITS)) & (N-1) - * - * sbo_find_obj_node(meta, offset) → (box, slot): - * unit = offset / 8 - * root_si = offset / slot_bytes - * node = root box of that slot - * cur_level = node.objlevel - * while cur_level >= 1: - * slot = (unit >> ((cur_level-1)*N_BITS)) & (N-1) - * if node.slots[slot].state == OBJ_START: - * return (node, slot) - * if node.slots[slot].state == BOX: - * node = sbo_child(meta, node, node.children[slot]) - * cur_level-- - * continue - * return NULL - * return NULL - */ - -/* --- 3.8 offset 编码 --- - * - * sbo_offset_raw(level, slot): - * // slot 在 level-k box 内的字节偏移 (k≥1) - * return slot << ((level-1) * N_BITS + 3) - * - * sbo_offset_of(usage): - * // 总字节数 = multiple × 8 × N^(level-1) - * if usage.level == 0: return 8 - * return usage.multiple << ((usage.level-1) * N_BITS + 3) - */ - -/* --- 3.9 公共 API --- - * - * sbo_align_to(n_slots): - * // slot 数 → (level≥1, multiple∈[1,N-1]) - * // [1, N-1] → L1, [N, N(N-1)] → L2, ... - * // 找最小 k≥1 使 n_slots ≤ (N-1) × N^(k-1) - * - * sbo_init(metaptr, head_size, data_size): - * // data_size 必须 64-幂对齐 - * - * sbo_alloc(metaptr, size): - * 1. usage = sbo_align_to(ceil(size/8)) - * 2. hash → root_slot → trylock 轮询 - * 3. root box 子树容量 >= usage → sbo_find_alloc - * 4. total = si * slot_bytes + offset - * 5. 返回 total - * - * sbo_free(metaptr, obj_offset): - * 1. lock → sbo_find_obj_node → (box, slot) - * 2. sbo_free_obj_slots(box, slot) - * 3. unlock - * - * sbo_data_ptr(metaptr, data_base, obj_offset): - * // meta/data 分离, data_base = data 区起始 - * unit = obj_offset / 8 - * node = root; cur_level = node.objlevel; phys_off = 0 - * while cur_level >= 1: - * slot = (unit >> ((cur_level-1)*N_BITS)) & (N-1) - * phys_off += slot * 8 * N^(node.objlevel - 1) - * if node.slots[slot].state == OBJ_START: break - * node = sbo_child(meta, node, node.children[slot]) - * cur_level-- - * return data_base + phys_off - */ - - -/* ================================================================ - * 4. 碎片化处理（无 compact） - * ================================================================ - * - * alloc 路径: - * 1. 边界分配 (box_boundary / obj_boundary) ← O(1) 首选 - * 2. 失败 → 从边界向外扫描 free_bitmap ← O(N) 次选 - * 3. 仍失败 → 向上传播容量不足 - * - * free 路径: - * 1. 标记 FREE, bitmap_set - * 2. 不调整边界 (碎片留待后续 alloc 扫描吸收) - * - * N=64 的 64-slot 容量使 left-box/right-obj 的边界分配命中率很高， - * 短暂碎片被统计稀释，无需 compact。 - */ - - -/* ================================================================ - * 5. 关键不变量 - * ================================================================ - * - * 1. 左 box 右 obj: - * alloc_box_slot 始终向右扩展 box_boundary - * alloc_obj_slots 始终向左收缩 obj_boundary - * 新分配不会在空闲区中间"挖洞" - * - * 2. free_bitmap 与 slots 一致: - * bitmap_get(fb, i) == 1 ⇔ slots[i].state == FREE - * - * 3. 容量 hint 正确: - * sbo_box_and_child_max_cap 在 alloc/free 后更新并向上传播 - */ +/* slotsboxobj.h is authoritative; see doc/perf-v3.md for benchmarks. */ -#endif /* SBO_DOC_H */ +#endif diff --git a/include/slotsboxmalloc/slotsboxobj.h b/include/slotsboxmalloc/slotsboxobj.h index ce2b787..814506e 100644 --- a/include/slotsboxmalloc/slotsboxobj.h +++ b/include/slotsboxmalloc/slotsboxobj.h @@ -1,15 +1,5 @@ -/* - * slotsboxobj.h — N=64 left-box/right-obj 内存分配器 (header-only) - * - * 严格遵循 doc.h 模型：子 box 从左侧分配，对象从右侧分配，空闲居中。 - * 无 compact — N=64 容量统计吸收碎片。 - * - * 模型: L0=8B(无slot), L1=512B(64×8B), L2=32KB, L3=2MB, L4=128MB, ... - * 每 box 固定 64 个 slot。freebitmap = uint8_t[8]。 - * - * SHM 兼容：零指针，纯 block-id + offset。 - * 并发：per-slot spinlock (64B padded)，trylock 轮询。 - */ +/* N=64 buddy allocator, meta format v3. Meta memory must be trusted; + * a process killed while holding a shard lock leaves that shard locked. */ #ifndef SLOTSBOXOBJ_H #define SLOTSBOXOBJ_H @@ -20,136 +10,122 @@ #include #include -/* ================================================================ - * 常量 - * ================================================================ */ - #define SBO_N 64 #define SBO_N_BITS 6 #define SBO_N_MASK 0x3F #define SBO_MULT_MAX 63 -#define SBO_BITMAP_B (SBO_N / 8) -#define SBO_MAX_ROOT 64 -#define SBO_MAGIC "slotsboxobj64v2" +#define SBO_MAGIC "slotsboxobj64v3" + +#ifndef SBO_MIN_SHARD_BYTES +#define SBO_MIN_SHARD_BYTES (8ULL * 64 * 64 * 64) +#endif +#define SBO_MAX_SHARDS 64 -/* ================================================================ - * obj_usage — 8 × 64^(level-1) × multiple bytes (level≥1) - * ================================================================ */ +/* Capacity: 8 * 64^(level-1) * multiple bytes. */ typedef struct { uint8_t level; uint8_t multiple; -} __attribute__((packed)) sbo_usage_t; - -static inline uint64_t sbo_pow64(uint32_t lvl) { - return (uint64_t)1 << (lvl * SBO_N_BITS); -} +} sbo_usage_t; -static inline uint32_t sbo_log64(uint64_t n) { - return (uint32_t)((63 - __builtin_clzll(n)) / SBO_N_BITS); +static inline uint16_t sbo_enc(uint8_t level, uint8_t multiple) { + return (uint16_t)(((uint16_t)level << 6) | multiple); } +static inline uint16_t sbo_enc_of(sbo_usage_t u) { return sbo_enc(u.level, u.multiple); } static inline uint64_t sbo_offset_raw(uint8_t level, uint8_t slot) { + if (level == 0) return (uint64_t)slot << 3; return (uint64_t)slot << ((level - 1) * SBO_N_BITS + 3); } -static inline uint64_t sbo_offset_of(sbo_usage_t u) { - if (u.level == 0) return 8; - return (uint64_t)u.multiple << ((u.level - 1) * SBO_N_BITS + 3); -} - -static inline int sbo_usage_cmp(sbo_usage_t a, sbo_usage_t b) { - uint16_t pa = (uint16_t)((a.level << 6) | a.multiple); - uint16_t pb = (uint16_t)((b.level << 6) | b.multiple); - return (int)pa - (int)pb; -} - static inline sbo_usage_t sbo_align_to(uint64_t n_slots) { sbo_usage_t r = {0, 0}; if (n_slots == 0) return r; if (n_slots <= SBO_MULT_MAX) { r.level = 1; r.multiple = (uint8_t)n_slots; return r; } - uint8_t k = 1; - uint64_t limit = SBO_MULT_MAX; - while (n_slots > limit) { k++; limit = (uint64_t)SBO_MULT_MAX * sbo_pow64(k - 1); } - uint64_t base = sbo_pow64(k - 1); - r.level = k; - r.multiple = (uint8_t)((n_slots + base - 1) / base); + uint64_t t = n_slots / SBO_MULT_MAX + (n_slots % SBO_MULT_MAX != 0); + uint32_t k = (uint32_t)((64 - __builtin_clzll(t - 1) + SBO_N_BITS - 1) / SBO_N_BITS); + uint32_t sh = k * SBO_N_BITS; + r.level = (uint8_t)(k + 1); + r.multiple = (uint8_t)((n_slots >> sh) + ((n_slots & ((1ULL << sh) - 1)) != 0)); return r; } -/* ================================================================ - * Spinlock (cache-line padded) - * ================================================================ */ - -typedef struct { atomic_int lock; uint8_t _pad[60]; } sbo_lock_t; - -static inline void sbo_lock(sbo_lock_t *lk) { - int e = 0; - while (!atomic_compare_exchange_weak(&lk->lock, &e, 1)) { - e = 0; +typedef struct { + atomic_int lock; + uint32_t hot_id1[3]; + uint64_t hot_base[3]; + uint8_t _pad[24]; +} sbo_lock_t; +_Static_assert(ATOMIC_INT_LOCK_FREE == 2, "atomic_int must be lock-free"); +_Static_assert(sizeof(sbo_lock_t) == 64, "sbo_lock_t must be 64 bytes"); + +static inline void sbo_cpu_relax(void) { #ifdef __x86_64__ - __asm__ __volatile__("pause" ::: "memory"); + __asm__ __volatile__("pause" ::: "memory"); #elif defined(__aarch64__) - __asm__ __volatile__("yield" ::: "memory"); + __asm__ __volatile__("yield" ::: "memory"); #endif - } } -static inline bool sbo_trylock(sbo_lock_t *lk) { int e = 0; return atomic_compare_exchange_weak(&lk->lock, &e, 1); } -static inline void sbo_unlock(sbo_lock_t *lk) { atomic_store(&lk->lock, 0); } -/* ================================================================ - * Slot / Box 状态 - * ================================================================ */ - -typedef enum { SBO_FREE = 0, SBO_BOX = 1, SBO_OBJ_START = 2, SBO_OBJ_CONT = 3 } sbo_state_e; - -typedef struct { uint8_t state : 2; uint8_t _rsv : 6; } __attribute__((packed)) sbo_slot_t; +static inline void sbo_lock(sbo_lock_t *lk) { + for (;;) { + if (atomic_load_explicit(&lk->lock, memory_order_relaxed) == 0) { + int e = 0; + if (atomic_compare_exchange_weak_explicit(&lk->lock, &e, 1, + memory_order_acquire, memory_order_relaxed)) + return; + } + sbo_cpu_relax(); + } +} +static inline bool sbo_trylock(sbo_lock_t *lk) { + if (atomic_load_explicit(&lk->lock, memory_order_relaxed) != 0) return false; + int e = 0; + return atomic_compare_exchange_weak_explicit(&lk->lock, &e, 1, + memory_order_acquire, memory_order_relaxed); +} +static inline void sbo_unlock(sbo_lock_t *lk) { + atomic_store_explicit(&lk->lock, 0, memory_order_release); +} -/* ================================================================ - * box_head_t — N=64 树节点 (packed, ~600B with bitmap+children) - * ================================================================ */ +/* Slot state: 0=free, 1=box, 0x40|(len-1)=object start. */ +#define SBO_FREE 0 +#define SBO_BOX 1 +#define SBO_START(len) ((uint8_t)(0x40 | ((len) - 1))) +#define SBO_IS_START(s) ((unsigned)((s) - 0x40) < SBO_MULT_MAX) +#define SBO_START_LEN(s) ((int)((s) & 0x3F) + 1) typedef struct { - uint8_t state; // SBO_BOX - uint8_t max_obj_cap; // 最大连续空闲 [0,64] - uint8_t slot_id; // root slot - int32_t parent; // 父 block id (0=root, -1=无) - uint8_t objlevel; // box level (≥1) - uint8_t _pad0; - - int16_t box_boundary; // 子 box 右边界: boxes ∈ [0, box_b), 满时=64 - int16_t obj_boundary; // 对象左边界: objs ∈ (obj_b, 63], 填到 slot0 时=-1 - - sbo_usage_t child_max_cap; // 子树容量 hint - uint8_t free_bitmap[SBO_BITMAP_B]; // bit=1→FREE - sbo_slot_t slots[64]; - int32_t children[64]; // 子 box block id (-1=无) -} __attribute__((packed)) sbo_box_t; - -/* ================================================================ - * box_meta_t — 全局元信息（base + 后跟 slot_pools + slot_locks） - * ================================================================ */ + uint64_t free_mask; + uint64_t child_mask; + int32_t parent; + int32_t self_id; + uint16_t child_max_enc; + uint8_t objlevel; + uint8_t max_obj_cap; + uint8_t hint_child; + uint8_t parent_slot; + uint8_t child_max_count; + uint8_t _pad; + uint8_t states[SBO_N]; + uint16_t child_cap[SBO_N]; + int32_t children[SBO_N]; +} sbo_box_t; typedef struct { uint8_t magic[16]; - uint64_t head_size; uint64_t data_size; - uint64_t slot_bytes; uint64_t per_slot_meta; - uint8_t root_slots; - uint8_t _pad[7]; + uint32_t next_shard; uint16_t block_stride; - uint8_t sizeof_block_head; - uint8_t _pad2[5]; - // 后跟: blocks_meta_t slot_pools[root_slots] + sbo_lock_t slot_locks[root_slots] + uint8_t root_slots; + uint8_t shard_mask; + uint8_t slot_shift; + uint8_t box_off; + uint32_t max_blocks; + uint8_t _pad[2]; } sbo_meta_t; -#define SBO_META_BASE sizeof(sbo_meta_t) - -/* ================================================================ - * Public API - * ================================================================ */ - int sbo_init(void *metaptr, size_t head_size, size_t data_size); uint64_t sbo_alloc(void *metaptr, size_t size); void sbo_free(void *metaptr, uint64_t obj_offset); @@ -157,57 +133,13 @@ uint64_t sbo_allocated_size(void *metaptr, uint64_t obj_offset); void *sbo_data_ptr(void *metaptr, void *data_base, uint64_t obj_offset); size_t sbo_meta_size(size_t data_size, size_t per_slot_pool); -#endif +#endif /* SLOTSBOXOBJ_H */ #ifdef SLOTSBOXMALLOC_IMPLEMENTATION #include -/* ================================================================ - * free_bitmap helpers (64-bit word + byte-oriented) - * ================================================================ */ - -static inline uint64_t sbo_bm_load(const uint8_t *bm) { - uint64_t v = 0; - for (int i = 0; i < SBO_BITMAP_B; i++) - v |= (uint64_t)bm[i] << (8 * i); - return v; -} -static inline void sbo_bm_store(uint8_t *bm, uint64_t v) { - for (int i = 0; i < SBO_BITMAP_B; i++) - bm[i] = (uint8_t)(v >> (8 * i)); -} -static inline uint64_t sbo_bm_range_mask(int start, int count) { - if (count <= 0 || start < 0 || start >= SBO_N) return 0; - if (count >= SBO_N - start) return UINT64_MAX << start; - return ((UINT64_C(1) << count) - 1) << start; -} -static inline bool sbo_bm_get(uint8_t *bm, int i) { - return (bm[i >> 3] >> (i & 7)) & 1; -} -static inline void sbo_bm_set(uint8_t *bm, int i) { - bm[i >> 3] |= (uint8_t)(1 << (i & 7)); -} -static inline void sbo_bm_clear(uint8_t *bm, int i) { - bm[i >> 3] &= (uint8_t)~(1 << (i & 7)); -} -static inline void sbo_bm_fill(uint8_t *bm) { - memset(bm, 0xFF, SBO_BITMAP_B); -} -static inline void sbo_bm_clear_range(uint8_t *bm, int start, int count) { - sbo_bm_store(bm, sbo_bm_load(bm) & ~sbo_bm_range_mask(start, count)); -} -static inline void sbo_bm_set_range(uint8_t *bm, int start, int count) { - sbo_bm_store(bm, sbo_bm_load(bm) | sbo_bm_range_mask(start, count)); -} -static inline int sbo_bm_popcount(uint8_t *bm) { - return __builtin_popcountll(sbo_bm_load(bm)); -} - -/* ================================================================ - * 内部 helper — meta 区布局: - * [sbo_meta_t][N×blocks_meta_t][N×sbo_lock_t][slot0_pool][slot1_pool]... - * ================================================================ */ +#define SBO_POOL_HDR (sizeof(blocks_meta_t) + sizeof(sbo_lock_t)) static inline blocks_meta_t *sbo_pool_meta(sbo_meta_t *meta, uint8_t si) { return (blocks_meta_t *)((uint8_t *)(meta + 1)) + si; @@ -215,30 +147,44 @@ static inline blocks_meta_t *sbo_pool_meta(sbo_meta_t *meta, uint8_t si) { static inline sbo_lock_t *sbo_pool_lock(sbo_meta_t *meta, uint8_t si) { return (sbo_lock_t *)((uint8_t *)(meta + 1) + meta->root_slots * sizeof(blocks_meta_t)) + si; } -static inline void *sbo_pool_mem(sbo_meta_t *meta, uint8_t si) { +static inline uint8_t *sbo_pool_mem(sbo_meta_t *meta, uint8_t si) { return (uint8_t *)(meta + 1) - + meta->root_slots * (sizeof(blocks_meta_t) + sizeof(sbo_lock_t)) + + meta->root_slots * SBO_POOL_HDR + si * meta->per_slot_meta; } -static inline sbo_box_t *sbo_child(sbo_meta_t *meta, sbo_box_t *box, int32_t cid) { - if (cid < 0) return NULL; - return (sbo_box_t *)((uint8_t *)sbo_pool_mem(meta, box->slot_id) - + cid * meta->block_stride + meta->sizeof_block_head); -} -static inline int32_t sbo_self_id(sbo_meta_t *meta, sbo_box_t *box) { - return blockid_bydataoffset(sbo_pool_meta(meta, box->slot_id), - (uint8_t *)box - (uint8_t *)sbo_pool_mem(meta, box->slot_id)); -} - -/* ================================================================ - * 容量查询 - * ================================================================ */ -static inline uint8_t sbo_bm_continuous_max(uint64_t m) { +typedef struct { + blocks_meta_t *pool_meta; + sbo_lock_t *lk; + uint8_t *pool; + uint32_t stride; + uint32_t head; + uint32_t max_blocks; + int32_t last_id; +} sbo_ctx_t; + +static inline sbo_ctx_t sbo_ctx_for(sbo_meta_t *meta, uint8_t si, sbo_lock_t *lk) { + sbo_ctx_t c; + c.pool_meta = sbo_pool_meta(meta, si); + c.lk = lk; + c.pool = sbo_pool_mem(meta, si); + c.stride = meta->block_stride; + c.head = meta->box_off; + c.max_blocks = meta->max_blocks; + c.last_id = -1; + return c; +} +static inline sbo_box_t *sbo_box_at(const sbo_ctx_t *c, int32_t id) { + return (sbo_box_t *)(c->pool + (uint32_t)id * c->stride + c->head); +} + +static inline uint8_t sbo_mask_contmax(uint64_t m) { if (!m) return 0; uint64_t u = m + (m & (~m + 1ULL)); - if (__builtin_expect((u & (u - 1ULL)) == 0, 1)) - return (uint8_t)(SBO_N - __builtin_clzll(m) - __builtin_ctzll(m)); + if (__builtin_expect((u & (u - 1ULL)) == 0, 1)) { + int n = 64 - __builtin_clzll(m) - __builtin_ctzll(m); + return (uint8_t)(n > SBO_MULT_MAX ? SBO_MULT_MAX : n); + } uint64_t m2 = m & (m >> 1); uint64_t m4 = m2 & (m2 >> 2); uint64_t m8 = m4 & (m4 >> 4); @@ -246,367 +192,534 @@ static inline uint8_t sbo_bm_continuous_max(uint64_t m) { uint64_t m32 = m16 & (m16 >> 16); uint8_t best = 1; uint64_t x = m, y; - y = (x >> 32) & m32; if (y) { x = y; best += 32; } + if (m32) { x = m32; best += 32; } y = (x >> 16) & m16; if (y) { x = y; best += 16; } y = (x >> 8) & m8; if (y) { x = y; best += 8; } y = (x >> 4) & m4; if (y) { x = y; best += 4; } y = (x >> 2) & m2; if (y) { x = y; best += 2; } y = (x >> 1) & m; if (y) { best += 1; } - return best; -} - -static inline uint8_t sbo_continuous_max(sbo_box_t *box) { - return sbo_bm_continuous_max(sbo_bm_load(box->free_bitmap)); + return best > SBO_MULT_MAX ? SBO_MULT_MAX : best; } -static inline sbo_usage_t sbo_box_max_cap(sbo_box_t *box) { - if (box->max_obj_cap == 0) return (sbo_usage_t){0, 0}; - if (box->max_obj_cap == 64) return (sbo_usage_t){box->objlevel + 1, 1}; - return (sbo_usage_t){box->objlevel, box->max_obj_cap}; +static inline int sbo_mask_find_run(uint64_t m, int need) { + uint64_t x = m; + int len = 1; + while (len * 2 <= need) { x &= (x >> len); len *= 2; } + if (len < need) x &= (x >> (need - len)); + return x ? (63 - __builtin_clzll(x)) : -1; } -static inline sbo_usage_t sbo_box_and_child_max_cap(sbo_box_t *box) { - sbo_usage_t own = sbo_box_max_cap(box); - return sbo_usage_cmp(own, box->child_max_cap) >= 0 ? own : box->child_max_cap; +static inline uint16_t sbo_combined(const sbo_box_t *box) { + uint16_t own = box->max_obj_cap ? sbo_enc(box->objlevel, box->max_obj_cap) : 0; + return own > box->child_max_enc ? own : box->child_max_enc; } -/* ================================================================ - * 容量向上传播 - * ================================================================ */ - -static inline void sbo_update_parent(sbo_meta_t *meta, sbo_box_t *box, - bool slot_c, bool child_c) { - if (slot_c) { - uint8_t nc = sbo_continuous_max(box); - if (box->max_obj_cap != nc) box->max_obj_cap = nc; else slot_c = false; +static inline void sbo_rescan_child_max(sbo_box_t *p) { + uint16_t mx = 0; + uint8_t cnt = 0; + uint64_t mm = p->child_mask; + while (mm) { + int i = __builtin_ctzll(mm); + mm &= mm - 1; + uint16_t v = p->child_cap[i]; + if (v > mx) { mx = v; cnt = 1; } + else if (v == mx) cnt++; } - if (child_c) { - sbo_usage_t nm = {0, 0}; - int lim = box->box_boundary < SBO_N ? box->box_boundary : SBO_N; - for (int i = 0; i < lim; i++) { - if (box->slots[i].state != SBO_BOX) continue; - sbo_box_t *ch = sbo_child(meta, box, box->children[i]); - if (!ch) continue; - sbo_usage_t cm = sbo_box_and_child_max_cap(ch); - if (sbo_usage_cmp(cm, nm) > 0) nm = cm; + p->child_max_enc = mx; + p->child_max_count = cnt; +} + +static inline void sbo_propagate(const sbo_ctx_t *c, sbo_box_t *box) { + uint16_t comb = sbo_combined(box); + while (box->parent >= 0) { + sbo_box_t *p = sbo_box_at(c, box->parent); + uint16_t old = p->child_cap[box->parent_slot]; + if (old == comb) return; + uint16_t p_old = sbo_combined(p); + p->child_cap[box->parent_slot] = comb; + if (comb > p->child_max_enc) { + p->child_max_enc = comb; + p->child_max_count = 1; + } else if (comb == p->child_max_enc) { + p->child_max_count++; + } else if (old == p->child_max_enc && --p->child_max_count == 0) { + sbo_rescan_child_max(p); } - if (sbo_usage_cmp(nm, box->child_max_cap) != 0) box->child_max_cap = nm; - else child_c = false; - } - if ((slot_c || child_c) && sbo_self_id(meta, box) != 0) { - sbo_box_t *p = sbo_child(meta, box, box->parent); - if (p) sbo_update_parent(meta, p, false, true); + comb = sbo_combined(p); + if (comb == p_old) return; + box = p; } } -/* ================================================================ - * box_format — 初始化 box - * ================================================================ */ - -static inline void sbo_box_format(sbo_meta_t *meta, sbo_box_t *box, - uint8_t objlevel, int32_t parent_id) { - box->state = SBO_BOX; - box->objlevel = objlevel; - box->parent = parent_id; - box->slot_id = 0; - box->box_boundary = 0; - box->obj_boundary = SBO_N - 1; - box->max_obj_cap = SBO_N; - sbo_bm_fill(box->free_bitmap); - for (int i = 0; i < SBO_N; i++) { - box->slots[i].state = SBO_FREE; - box->children[i] = -1; - } - box->child_max_cap = (sbo_usage_t){0, 0}; -} - -/* ================================================================ - * alloc_box_slot — 从左侧分配子 box slot - * ================================================================ */ - -static inline int8_t sbo_alloc_box_slot(sbo_meta_t *meta, sbo_box_t *box) { - (void)meta; - int lo = box->box_boundary > 0 ? box->box_boundary : 0; - int hi = box->obj_boundary < SBO_N - 1 ? box->obj_boundary : SBO_N - 1; - if (lo > hi || lo >= SBO_N || hi < 0) return -1; - uint64_t mask = (UINT64_MAX << lo) & (UINT64_MAX >> (SBO_N - 1 - hi)); - uint64_t runs = sbo_bm_load(box->free_bitmap) & mask; - if (!runs) return -1; - int p = __builtin_ctzll(runs); - sbo_bm_clear(box->free_bitmap, p); - box->slots[p].state = SBO_BOX; - box->box_boundary = (int16_t)(p + 1); - return (int8_t)p; -} - -/* ================================================================ - * alloc_obj_slots — 从右侧分配 m 个连续对象 slot - * ================================================================ */ - -static inline int8_t sbo_alloc_obj_slots(sbo_meta_t *meta, sbo_box_t *box, uint8_t m) { - if (m == 0 || m > SBO_MULT_MAX || m > box->max_obj_cap || box->box_boundary >= SBO_N) - return -1; - int lo = box->box_boundary > 0 ? box->box_boundary : 0; - uint64_t bm = sbo_bm_load(box->free_bitmap); - uint64_t runs = bm & (UINT64_MAX << lo); - /* 倍增移位与运算定位长度为 m 的连续空闲段起点，保留最右适配策略 */ - for (int width = 1; width < m;) { - int step = width < m - width ? width : m - width; - runs &= runs >> step; - width += step; - } - if (!runs) return -1; - int start = 63 - __builtin_clzll(runs); - bm &= ~sbo_bm_range_mask(start, m); - sbo_bm_store(box->free_bitmap, bm); - box->slots[start].state = SBO_OBJ_START; - for (int j = start + 1; j < start + m; j++) - box->slots[j].state = SBO_OBJ_CONT; - box->obj_boundary = (int16_t)(start - 1); - uint8_t nc = sbo_bm_continuous_max(bm); - if (box->max_obj_cap != nc) { +static inline void sbo_after_mask_change(const sbo_ctx_t *c, sbo_box_t *box) { + uint8_t old = box->max_obj_cap; + uint8_t nc = sbo_mask_contmax(box->free_mask); + if (nc != old) { box->max_obj_cap = nc; - if (sbo_self_id(meta, box) != 0) { - sbo_box_t *pp = sbo_child(meta, box, box->parent); - if (pp) sbo_update_parent(meta, pp, false, true); - } + sbo_propagate(c, box); } - return (int8_t)start; } -/* ================================================================ - * free_obj_slots — 释放对象 slot（不调边界） - * ================================================================ */ - -static inline void sbo_free_obj_slots(sbo_meta_t *meta, sbo_box_t *box, uint8_t start) { - int p = start; - box->slots[p].state = SBO_FREE; - p++; - while (p < SBO_N && box->slots[p].state == SBO_OBJ_CONT) { - box->slots[p].state = SBO_FREE; - p++; +static inline void sbo_box_format(sbo_box_t *box, uint8_t objlevel, + int32_t parent, uint8_t parent_slot, + int32_t self_id) { + box->free_mask = ~0ULL; + box->child_mask = 0; + box->parent = parent; + box->self_id = self_id; + box->child_max_enc = 0; + box->objlevel = objlevel; + box->max_obj_cap = SBO_MULT_MAX; + box->hint_child = 0; + box->parent_slot = parent_slot; + box->child_max_count = 0; + memset(box->states, 0, sizeof(box->states)); +} + +static bool sbo_reclaim_empty(const sbo_ctx_t *c, sbo_box_t *box) { + bool any = false; + uint64_t mm = box->child_mask; + while (mm) { + int i = __builtin_ctzll(mm); + mm &= mm - 1; + sbo_box_t *ch = sbo_box_at(c, box->children[i]); + if (ch->child_mask) sbo_reclaim_empty(c, ch); + if (ch->free_mask == ~0ULL && ch->child_mask == 0) { + int32_t cid = ch->self_id; + uint8_t lv = ch->objlevel; + if (lv >= 1 && lv <= 3 && c->lk->hot_id1[lv - 1] == (uint32_t)cid + 1) + c->lk->hot_id1[lv - 1] = 0; + blocks_free(c->pool_meta, c->pool, (uint64_t)cid); + box->child_mask &= ~(1ULL << i); + box->free_mask |= (1ULL << i); + box->states[i] = SBO_FREE; + box->child_cap[i] = 0; + any = true; + } } - uint64_t bm = sbo_bm_load(box->free_bitmap) | sbo_bm_range_mask(start, p - start); - sbo_bm_store(box->free_bitmap, bm); - uint8_t nc = sbo_bm_continuous_max(bm); - if (box->max_obj_cap != nc) { - box->max_obj_cap = nc; - if (sbo_self_id(meta, box) != 0) { - sbo_box_t *parent = sbo_child(meta, box, box->parent); - if (parent) sbo_update_parent(meta, parent, false, true); + if (any) { + sbo_rescan_child_max(box); + box->hint_child = box->child_mask ? (uint8_t)__builtin_ctzll(box->child_mask) : 0; + box->max_obj_cap = sbo_mask_contmax(box->free_mask); + sbo_propagate(c, box); + } + return any; +} + +static inline int sbo_alloc_obj_slots(sbo_ctx_t *c, sbo_box_t *box, uint8_t m) { + uint64_t fm = box->free_mask; + if (!fm) return -1; + uint64_t u = fm + (fm & (~fm + 1ULL)); + if (__builtin_expect((u & (u - 1ULL)) == 0, 1)) { + int hi = 64 - __builtin_clzll(fm); + int k = hi - __builtin_ctzll(fm); + if (k < (int)m) return -1; + int s = hi - (int)m; + box->free_mask = fm & ~(((1ULL << m) - 1ULL) << s); + box->states[s] = SBO_START(m); + c->last_id = box->self_id; + int rem = k - (int)m; + uint8_t nc = (uint8_t)(rem > SBO_MULT_MAX ? SBO_MULT_MAX : rem); + if (nc != box->max_obj_cap) { + box->max_obj_cap = nc; + sbo_propagate(c, box); } + return s; } + int s = sbo_mask_find_run(fm, m); + if (s < 0) return -1; + box->free_mask = fm & ~(((1ULL << m) - 1ULL) << s); + box->states[s] = SBO_START(m); + c->last_id = box->self_id; + sbo_after_mask_change(c, box); + return s; } -/* ================================================================ - * sbo_find_alloc — 递归树下降分配 - * ================================================================ */ +static inline void sbo_free_obj_slots(const sbo_ctx_t *c, sbo_box_t *box, uint8_t start) { + int len = SBO_START_LEN(box->states[start]); + if (start + len > SBO_N) return; + box->states[start] = SBO_FREE; + box->free_mask |= (((1ULL << len) - 1ULL) << start); + sbo_after_mask_change(c, box); +} #define SBO_ALLOC_FAIL ((uint64_t)-1) -static inline uint64_t sbo_find_alloc(sbo_meta_t *meta, sbo_box_t *box, - sbo_usage_t usage) { - if (!box || box->state != SBO_BOX) return SBO_ALLOC_FAIL; - +static uint64_t sbo_find_alloc(sbo_ctx_t *c, sbo_box_t *box, + sbo_usage_t usage, uint16_t need_enc) { if (usage.level == box->objlevel) { - if (usage.multiple > box->max_obj_cap) return SBO_ALLOC_FAIL; - int8_t slot = sbo_alloc_obj_slots(meta, box, usage.multiple); + int slot = sbo_alloc_obj_slots(c, box, usage.multiple); + if (slot < 0 && box->child_mask && sbo_reclaim_empty(c, box)) + slot = sbo_alloc_obj_slots(c, box, usage.multiple); if (slot < 0) return SBO_ALLOC_FAIL; return sbo_offset_raw(box->objlevel, (uint8_t)slot); } - - if (usage.level < box->objlevel) { - // 先遍历已有子 box - if (sbo_usage_cmp(box->child_max_cap, usage) >= 0) { - int lim = box->box_boundary < SBO_N ? box->box_boundary : SBO_N; - for (int i = 0; i < lim; i++) { - if (box->children[i] < 0) continue; - sbo_box_t *ch = sbo_child(meta, box, box->children[i]); - if (!ch) continue; - if (sbo_usage_cmp(sbo_box_and_child_max_cap(ch), usage) < 0) continue; - uint64_t sub = sbo_find_alloc(meta, ch, usage); - if (sub != SBO_ALLOC_FAIL) + if (usage.level > box->objlevel) return SBO_ALLOC_FAIL; + + if (box->child_max_enc >= need_enc) { + int start = box->hint_child; + for (int part = 0; part < 2; part++) { + uint64_t mm = (part == 0) + ? (box->child_mask & (~0ULL << start)) + : (box->child_mask & ((start == 0) ? 0ULL : ((1ULL << start) - 1))); + while (mm) { + int i = __builtin_ctzll(mm); + mm &= mm - 1; + if (box->child_cap[i] < need_enc) continue; + uint64_t sub = sbo_find_alloc(c, sbo_box_at(c, box->children[i]), + usage, need_enc); + if (sub != SBO_ALLOC_FAIL) { + if (box->hint_child != (uint8_t)i) box->hint_child = (uint8_t)i; return sbo_offset_raw(box->objlevel, (uint8_t)i) + sub; + } } } - // 无已有子 box 满足 → 从左侧分配新子 box slot - int8_t slot = sbo_alloc_box_slot(meta, box); - if (slot < 0) return SBO_ALLOC_FAIL; + } - int64_t cid = blocks_alloc(sbo_pool_meta(meta, box->slot_id), - sbo_pool_mem(meta, box->slot_id)); - if (cid < 0) { - box->slots[slot].state = SBO_FREE; - sbo_bm_set(box->free_bitmap, slot); + uint32_t need_blk = (uint32_t)(box->objlevel - usage.level); + if (!box->free_mask || c->max_blocks - (uint32_t)c->pool_meta->used_blocks < need_blk) { + sbo_reclaim_empty(c, sbo_box_at(c, 0)); + if (!box->free_mask || c->max_blocks - (uint32_t)c->pool_meta->used_blocks < need_blk) return SBO_ALLOC_FAIL; - } - box->children[slot] = (int32_t)cid; - sbo_box_t *ch = sbo_child(meta, box, cid); - sbo_box_format(meta, ch, box->objlevel - 1, sbo_self_id(meta, box)); - ch->slot_id = box->slot_id; - - sbo_update_parent(meta, box, true, true); - - sbo_usage_t ch_max = {ch->objlevel + 1, 1}; - if (sbo_usage_cmp(ch_max, usage) >= 0) { - uint64_t sub = sbo_find_alloc(meta, ch, usage); - if (sub != SBO_ALLOC_FAIL) - return sbo_offset_raw(box->objlevel, (uint8_t)slot) + sub; - } } - return SBO_ALLOC_FAIL; -} -/* ================================================================ - * sbo_find_obj_node — offset → (box, slot_index) - * 无 compact: physical == virtual, 直接 6-bit 解码 - * ================================================================ */ + int64_t cid = blocks_alloc(c->pool_meta, c->pool); + if (cid < 0) return SBO_ALLOC_FAIL; + int slot = __builtin_ctzll(box->free_mask); + box->free_mask &= ~(1ULL << slot); + box->states[slot] = SBO_BOX; + box->children[slot] = (int32_t)cid; + sbo_box_t *ch = sbo_box_at(c, (int32_t)cid); + sbo_box_format(ch, (uint8_t)(box->objlevel - 1), box->self_id, + (uint8_t)slot, (int32_t)cid); + uint16_t ch_comb = sbo_combined(ch); + box->child_cap[slot] = ch_comb; + box->child_mask |= (1ULL << slot); + if (ch_comb > box->child_max_enc) { + box->child_max_enc = ch_comb; + box->child_max_count = 1; + } else if (ch_comb == box->child_max_enc) { + box->child_max_count++; + } + box->max_obj_cap = sbo_mask_contmax(box->free_mask); + + uint64_t sub = sbo_find_alloc(c, ch, usage, need_enc); + if (sub == SBO_ALLOC_FAIL) { + blocks_free(c->pool_meta, c->pool, (uint64_t)cid); + box->child_mask &= ~(1ULL << slot); + box->free_mask |= (1ULL << slot); + box->states[slot] = SBO_FREE; + box->child_cap[slot] = 0; + sbo_rescan_child_max(box); + box->max_obj_cap = sbo_mask_contmax(box->free_mask); + sbo_propagate(c, box); + return SBO_ALLOC_FAIL; + } + if (box->hint_child != (uint8_t)slot) box->hint_child = (uint8_t)slot; + return sbo_offset_raw(box->objlevel, (uint8_t)slot) + sub; +} -static inline sbo_box_t *sbo_find_obj_node(sbo_meta_t *meta, uint64_t obj_offset, - uint8_t *out_slot, uint8_t *out_root) { +static inline sbo_box_t *sbo_find_obj_node(const sbo_ctx_t *c, sbo_box_t *root, + uint64_t obj_offset, uint8_t *out_slot) { + if (root->objlevel == 0 || root->objlevel > 10) return NULL; uint64_t unit = obj_offset >> 3; - uint8_t si = (uint8_t)(obj_offset / meta->slot_bytes); - if (si >= meta->root_slots) return NULL; - *out_root = si; - - sbo_box_t *node = (sbo_box_t *)((uint8_t *)sbo_pool_mem(meta, si) - + meta->sizeof_block_head); - if (!node || node->state != SBO_BOX) return NULL; - + sbo_box_t *node = root; int level = node->objlevel; while (level >= 1) { - uint8_t slot = (uint8_t)((unit >> ((level - 1) * SBO_N_BITS)) & SBO_N_MASK); - if (node->slots[slot].state == SBO_OBJ_START) { *out_slot = slot; return node; } - if (node->slots[slot].state == SBO_BOX && node->children[slot] >= 0) { - node = sbo_child(meta, node, node->children[slot]); + uint32_t sh = (uint32_t)(level - 1) * SBO_N_BITS; + uint8_t slot = (uint8_t)((unit >> sh) & SBO_N_MASK); + if ((node->child_mask >> slot) & 1ULL) { + node = sbo_box_at(c, node->children[slot]); level--; continue; } + uint8_t st = node->states[slot]; + if (SBO_IS_START(st) && (unit & ((1ULL << sh) - 1)) == 0 + && slot + SBO_START_LEN(st) <= SBO_N) { + *out_slot = slot; + return node; + } return NULL; } return NULL; } -/* ================================================================ - * Public API - * ================================================================ */ - -int sbo_init(void *metaptr, size_t head_size, size_t data_size) { - if (!metaptr || data_size % 8 != 0) return -1; - sbo_meta_t *meta = (sbo_meta_t *)metaptr; - if (memcmp(meta->magic, SBO_MAGIC, sizeof(SBO_MAGIC) - 1) == 0) return -1; - - // data_size 必须 = 8 × 64^k (k ≥ 0) - uint64_t slots = data_size / 8; - if (slots == 0 || (slots & (slots - 1)) != 0) return -1; // 非 2 的幂 - if (__builtin_ctzll(slots) % SBO_N_BITS != 0) return -1; // 非 64 的幂 +static inline uint8_t sbo_shards_for_data(size_t data_size) { + return (data_size / SBO_MAX_SHARDS >= SBO_MIN_SHARD_BYTES) + ? (uint8_t)SBO_MAX_SHARDS : (uint8_t)1; +} - sbo_usage_t rounded = sbo_align_to(slots); +static inline bool sbo_magic_ok(const sbo_meta_t *meta) { + uint64_t got0, got1, want0, want1; + memcpy(&got0, meta->magic, 8); + memcpy(&got1, meta->magic + 8, 8); + memcpy(&want0, SBO_MAGIC, 8); + memcpy(&want1, SBO_MAGIC + 8, 8); + return ((got0 ^ want0) | (got1 ^ want1)) == 0; +} - meta->head_size = head_size; - meta->data_size = data_size; +static inline bool sbo_meta_ok(const sbo_meta_t *meta) { + if (!sbo_magic_ok(meta)) return false; + if (meta->root_slots != 1 && meta->root_slots != SBO_MAX_SHARDS) return false; + if (meta->shard_mask != meta->root_slots - 1) return false; + if (meta->slot_shift < 9 || meta->slot_shift >= 64) return false; + if (meta->data_size == 0 + || (meta->data_size >> meta->slot_shift) != meta->root_slots) return false; + if ((meta->block_stride & 7) != 0 || (meta->box_off & 7) != 0) return false; + if (meta->box_off > meta->block_stride + || (size_t)(meta->block_stride - meta->box_off) < sizeof(sbo_box_t)) return false; + if (meta->per_slot_meta < meta->block_stride) return false; + return true; +} - uint8_t root_slots = rounded.multiple; - size_t header_base = sizeof(sbo_meta_t) - + (size_t)root_slots * (sizeof(blocks_meta_t) + sizeof(sbo_lock_t)); - size_t per_slot_meta = (head_size - header_base) / root_slots; - meta->root_slots = root_slots; - meta->slot_bytes = data_size / root_slots; - meta->per_slot_meta = per_slot_meta; +#ifndef SBO_NO_TLS +typedef struct { const void *meta; int16_t home; uint16_t miss; } sbo_tls_t; +static __thread sbo_tls_t sbo_tls __attribute__((tls_model("initial-exec"))) = {NULL, -1, 0}; +#endif - for (uint8_t si = 0; si < root_slots; si++) { - atomic_init(&sbo_pool_lock(meta, si)->lock, 0); - void *pool_mem = sbo_pool_mem(meta, si); - blocks_init(sbo_pool_meta(meta, si), per_slot_meta, sizeof(sbo_box_t)); +static inline int sbo_next_home(sbo_meta_t *meta) { + return (int)((__atomic_fetch_add(&meta->next_shard, 1u, __ATOMIC_RELAXED) * 9u) + & meta->shard_mask); +} + +static inline int sbo_home_shard(sbo_meta_t *meta) { + if (meta->root_slots == 1) return 0; +#ifdef SBO_NO_TLS + return sbo_next_home(meta); +#else + if (sbo_tls.meta != (const void *)meta || sbo_tls.home < 0 + || sbo_tls.home > meta->shard_mask) { + sbo_tls.meta = (const void *)meta; + sbo_tls.home = (int16_t)sbo_next_home(meta); + sbo_tls.miss = 0; + } + return sbo_tls.home; +#endif +} - int64_t rid = blocks_alloc(sbo_pool_meta(meta, si), pool_mem); - if (rid < 0) return -1; +#define SBO_MIN_POOL_BLOCKS 32 - sbo_box_t *root = (sbo_box_t *)((uint8_t *)pool_mem - + blockdata_offset(sbo_pool_meta(meta, si), rid)); - sbo_box_format(meta, root, rounded.level - 1, 0); - root->slot_id = si; +int sbo_init(void *metaptr, size_t head_size, size_t data_size) { + if (!metaptr || ((uintptr_t)metaptr & 7) != 0) return -1; + if (head_size <= sizeof(sbo_meta_t) + SBO_POOL_HDR || (data_size & 7) != 0) return -1; + sbo_meta_t *meta = (sbo_meta_t *)metaptr; + if (sbo_magic_ok(meta)) return -1; + + uint64_t slots = data_size >> 3; + if (slots < SBO_N || (slots & (slots - 1)) != 0) return -1; + if (__builtin_ctzll(slots) % SBO_N_BITS != 0) return -1; + + uint8_t shards = sbo_shards_for_data(data_size); + size_t header_base, per_slot_meta; + for (;;) { + header_base = sizeof(sbo_meta_t) + (size_t)shards * SBO_POOL_HDR; + if (head_size <= header_base) return -1; + per_slot_meta = ((head_size - header_base) / shards) & ~(size_t)7; + if (shards == 1 + || per_slot_meta >= (size_t)SBO_MIN_POOL_BLOCKS * (sizeof(sbo_box_t) + 16)) + break; + shards = 1; } + if (per_slot_meta < sizeof(sbo_box_t) + 16) return -1; - meta->sizeof_block_head = (uint8_t)sbo_pool_meta(meta, 0)->sizeof_block_head; - meta->block_stride = (uint16_t)(meta->sizeof_block_head + sizeof(sbo_box_t)); + memset(meta, 0, sizeof(*meta)); + meta->data_size = data_size; + meta->root_slots = shards; + meta->shard_mask = (uint8_t)(shards - 1); + meta->slot_shift = (uint8_t)__builtin_ctzll(data_size / shards); + meta->per_slot_meta = per_slot_meta; + memset((uint8_t *)(meta + 1), 0, (size_t)shards * SBO_POOL_HDR); + + uint8_t root_level = + (uint8_t)(sbo_align_to(slots / shards).level - 1); + for (uint8_t si = 0; si < shards; si++) { + blocks_meta_t *pm = sbo_pool_meta(meta, si); + uint8_t *pool = sbo_pool_mem(meta, si); + uint64_t bs = sizeof(sbo_box_t); + uint8_t head = 0, pad = 0; + for (int it = 0; it < 4; it++) { + blocks_init(pm, per_slot_meta, bs); + head = (uint8_t)pm->sizeof_block_head; + pad = (uint8_t)((8 - (head & 7)) & 7); + uint64_t nbs = pad + ((sizeof(sbo_box_t) + 7) & ~(uint64_t)7); + if (nbs == bs) break; + bs = nbs; + } + if (si == 0) { + meta->box_off = (uint8_t)(head + pad); + meta->block_stride = (uint16_t)(head + bs); + if ((meta->block_stride & 7) != 0 || (meta->box_off & 7) != 0) + return -1; + meta->max_blocks = (uint32_t)(per_slot_meta / meta->block_stride); + } + if (blocks_alloc(pm, pool) != 0) return -1; + sbo_box_format((sbo_box_t *)(pool + meta->box_off), root_level, -1, 255, 0); + } - memset(meta->magic, 0, sizeof(meta->magic)); +#ifndef SBO_NO_TLS + if (sbo_tls.meta == (const void *)meta) { sbo_tls.meta = NULL; sbo_tls.home = -1; sbo_tls.miss = 0; } +#endif memcpy(meta->magic, SBO_MAGIC, sizeof(SBO_MAGIC) - 1); return 0; } +static bool sbo_try_hint(sbo_ctx_t *c, sbo_box_t *root, sbo_usage_t usage, + uint16_t need_enc, uint64_t *off_out) { + if (usage.level >= 1 && usage.level <= 3) { + uint32_t id1 = c->lk->hot_id1[usage.level - 1]; + if (id1 != 0) { + sbo_box_t *hb = sbo_box_at(c, (int32_t)(id1 - 1)); + if (hb->objlevel == usage.level && hb->max_obj_cap >= usage.multiple) { + int slot = sbo_alloc_obj_slots(c, hb, usage.multiple); + if (slot >= 0) { + *off_out = c->lk->hot_base[usage.level - 1] + + sbo_offset_raw(usage.level, (uint8_t)slot); + return true; + } + } + } + } + sbo_box_t *box = root; + uint64_t base = 0; + for (int guard = 0; guard < 16 && usage.level < box->objlevel; guard++) { + uint8_t h = box->hint_child; + if (h >= SBO_N) return false; + if ((box->child_mask & (1ULL << h)) == 0) return false; + if (box->child_cap[h] < need_enc || box->children[h] < 0) return false; + uint8_t parent_level = box->objlevel; + sbo_box_t *ch = sbo_box_at(c, box->children[h]); + if ((uint8_t)(ch->objlevel + 1) != parent_level) return false; + base += sbo_offset_raw(parent_level, h); + box = ch; + } + if (usage.level != box->objlevel) return false; + int slot = sbo_alloc_obj_slots(c, box, usage.multiple); + if (slot < 0) return false; + if (usage.level >= 1 && usage.level <= 3) { + c->lk->hot_id1[usage.level - 1] = (uint32_t)box->self_id + 1; + c->lk->hot_base[usage.level - 1] = base; + } + *off_out = base + sbo_offset_raw(box->objlevel, (uint8_t)slot); + return true; +} + uint64_t sbo_alloc(void *metaptr, size_t size) { if (!metaptr || size == 0) return SBO_ALLOC_FAIL; sbo_meta_t *meta = (sbo_meta_t *)metaptr; - if (memcmp(meta->magic, SBO_MAGIC, sizeof(SBO_MAGIC) - 1) != 0) - return SBO_ALLOC_FAIL; - - sbo_usage_t usage = sbo_align_to((size + 7) / 8); - int start = (int)((size ^ ((uint64_t)&size >> 4)) % meta->root_slots); - - for (int t = 0; t < meta->root_slots; t++) { - int si = (start + t) % meta->root_slots; - if (!sbo_trylock(sbo_pool_lock(meta, si))) continue; + if (!sbo_meta_ok(meta)) return SBO_ALLOC_FAIL; + uint64_t shard_bytes = 1ULL << meta->slot_shift; + if (size > shard_bytes - (shard_bytes >> SBO_N_BITS)) return SBO_ALLOC_FAIL; + + sbo_usage_t usage = sbo_align_to((size >> 3) + ((size & 7) != 0)); + uint16_t need_enc = sbo_enc_of(usage); + int shards = meta->root_slots; + int home = sbo_home_shard(meta); + bool home_full = false; + + for (int pass = 0; pass < 2; pass++) { + for (int t = 0; t < shards; t++) { + int si = (home + t) & meta->shard_mask; + sbo_lock_t *lk = sbo_pool_lock(meta, (uint8_t)si); + if (pass == 0) { if (!sbo_trylock(lk)) continue; } + else sbo_lock(lk); + + sbo_ctx_t c = sbo_ctx_for(meta, (uint8_t)si, lk); + sbo_box_t *root = sbo_box_at(&c, 0); + uint64_t off = SBO_ALLOC_FAIL; + if (sbo_combined(root) < need_enc && root->child_mask) + sbo_reclaim_empty(&c, root); + if (sbo_combined(root) >= need_enc) { + if (!sbo_try_hint(&c, root, usage, need_enc, &off)) { + off = sbo_find_alloc(&c, root, usage, need_enc); + if (off != SBO_ALLOC_FAIL && usage.level >= 1 && usage.level <= 3 + && c.last_id >= 0) { + uint32_t bsh = (uint32_t)usage.level * SBO_N_BITS + 3; + lk->hot_id1[usage.level - 1] = (uint32_t)c.last_id + 1; + lk->hot_base[usage.level - 1] = + bsh < 64 ? (off & ~((1ULL << bsh) - 1)) : 0; + } + } + } + sbo_unlock(lk); - sbo_box_t *root = (sbo_box_t *)((uint8_t *)sbo_pool_mem(meta, si) - + meta->sizeof_block_head); - if (sbo_usage_cmp(sbo_box_and_child_max_cap(root), usage) >= 0) { - uint64_t off = sbo_find_alloc(meta, root, usage); if (off != SBO_ALLOC_FAIL) { - uint64_t total = (uint64_t)si * meta->slot_bytes + off; - if (total + sbo_offset_of(usage) <= meta->data_size) { - sbo_unlock(sbo_pool_lock(meta, si)); - return total; +#ifndef SBO_NO_TLS + if (t == 0) { + sbo_tls.miss = 0; + } else if (sbo_tls.meta == (const void *)meta + && (home_full || ++sbo_tls.miss >= 4)) { + sbo_tls.home = (int16_t)si; + sbo_tls.miss = 0; } - uint8_t us = 0, ur = 0; - sbo_box_t *n = sbo_find_obj_node(meta, total, &us, &ur); - if (n) { n->slots[us].state = SBO_FREE; sbo_bm_set(n->free_bitmap, us); } +#endif + return ((uint64_t)si << meta->slot_shift) + off; } + if (t == 0) home_full = true; } - sbo_unlock(sbo_pool_lock(meta, si)); } return SBO_ALLOC_FAIL; } +static inline bool sbo_offset_ok(const sbo_meta_t *meta, uint64_t obj_offset) { + return sbo_meta_ok(meta) && (obj_offset & 7) == 0 + && obj_offset < meta->data_size + && (obj_offset >> meta->slot_shift) < meta->root_slots; +} + +static inline sbo_box_t *sbo_lookup_locked(sbo_meta_t *meta, uint64_t obj_offset, + sbo_ctx_t *c, sbo_lock_t **lk_out, + uint8_t *slot) { + uint8_t si = (uint8_t)(obj_offset >> meta->slot_shift); + sbo_lock_t *lk = sbo_pool_lock(meta, si); + *lk_out = lk; + sbo_lock(lk); + *c = sbo_ctx_for(meta, si, lk); + uint64_t local_off = obj_offset & ((1ULL << meta->slot_shift) - 1); + for (uint8_t lv = 1; lv <= 2; lv++) { + uint32_t id1 = lk->hot_id1[lv - 1]; + if (!id1) continue; + uint32_t sh = (uint32_t)(lv - 1) * SBO_N_BITS + 3; + uint64_t rel = local_off - lk->hot_base[lv - 1]; + if ((rel & ~((uint64_t)SBO_N_MASK << sh)) == 0) { + uint8_t s = (uint8_t)(rel >> sh); + sbo_box_t *hb = sbo_box_at(c, (int32_t)(id1 - 1)); + uint8_t st = hb->states[s]; + if (hb->objlevel == lv && SBO_IS_START(st) + && s + SBO_START_LEN(st) <= SBO_N) { + *slot = s; + return hb; + } + } + } + return sbo_find_obj_node(c, sbo_box_at(c, 0), local_off, slot); +} + void sbo_free(void *metaptr, uint64_t obj_offset) { if (!metaptr) return; sbo_meta_t *meta = (sbo_meta_t *)metaptr; - uint8_t root_si = (uint8_t)(obj_offset / meta->slot_bytes); - if (root_si >= meta->root_slots) return; - - sbo_lock(sbo_pool_lock(meta, root_si)); - uint8_t slot = 0, out_r = 0; - sbo_box_t *node = sbo_find_obj_node(meta, obj_offset, &slot, &out_r); - if (!node) { sbo_unlock(sbo_pool_lock(meta, root_si)); return; } - sbo_free_obj_slots(meta, node, slot); - sbo_unlock(sbo_pool_lock(meta, root_si)); + if (!sbo_offset_ok(meta, obj_offset)) return; + sbo_ctx_t c; + sbo_lock_t *lk; + uint8_t slot = 0; + sbo_box_t *node = sbo_lookup_locked(meta, obj_offset, &c, &lk, &slot); + if (node) sbo_free_obj_slots(&c, node, slot); + sbo_unlock(lk); } uint64_t sbo_allocated_size(void *metaptr, uint64_t obj_offset) { if (!metaptr) return 0; sbo_meta_t *meta = (sbo_meta_t *)metaptr; - uint8_t root_si = (uint8_t)(obj_offset / meta->slot_bytes); - if (root_si >= meta->root_slots) return 0; - - sbo_lock(sbo_pool_lock(meta, root_si)); - uint8_t slot = 0, out_r = 0; - sbo_box_t *node = sbo_find_obj_node(meta, obj_offset, &slot, &out_r); - if (!node) { sbo_unlock(sbo_pool_lock(meta, root_si)); return 0; } - - uint8_t count = 1; - for (int i = slot + 1; i < SBO_N && node->slots[i].state == SBO_OBJ_CONT; i++) count++; - - sbo_usage_t u; - if (count == 64) { u.level = node->objlevel + 1; u.multiple = 1; } - else { u.level = node->objlevel; u.multiple = count; } - uint64_t r = sbo_offset_of(u); - sbo_unlock(sbo_pool_lock(meta, root_si)); + if (!sbo_offset_ok(meta, obj_offset)) return 0; + sbo_ctx_t c; + sbo_lock_t *lk; + uint8_t slot = 0; + sbo_box_t *node = sbo_lookup_locked(meta, obj_offset, &c, &lk, &slot); + uint64_t r = 0; + if (node) r = sbo_offset_raw(node->objlevel, (uint8_t)SBO_START_LEN(node->states[slot])); + sbo_unlock(lk); return r; } -/* sbo_data_ptr: meta/data 分离。data_base = data 区起始地址。 - * 无 compact → physical==virtual, 直接 data_base + offset。 */ void *sbo_data_ptr(void *metaptr, void *data_base, uint64_t obj_offset) { (void)metaptr; if (!data_base) return NULL; @@ -614,9 +727,8 @@ void *sbo_data_ptr(void *metaptr, void *data_base, uint64_t obj_offset) { } size_t sbo_meta_size(size_t data_size, size_t per_slot_pool) { - sbo_usage_t r = sbo_align_to(data_size / 8); - return sizeof(sbo_meta_t) - + (size_t)r.multiple * (sizeof(blocks_meta_t) + sizeof(sbo_lock_t) + per_slot_pool); + uint8_t shards = sbo_shards_for_data(data_size); + return sizeof(sbo_meta_t) + (size_t)shards * (SBO_POOL_HDR + per_slot_pool); } #endif /* SLOTSBOXMALLOC_IMPLEMENTATION */ diff --git a/tutorial/01_basic.c b/tutorial/01_basic.c index e54d8be..4b92fa4 100644 --- a/tutorial/01_basic.c +++ b/tutorial/01_basic.c @@ -5,10 +5,9 @@ #include int main(void) { - size_t data_size = 8ULL * 64 * 64 * 64 * 64; // 16MB + size_t data_size = 8ULL * 64 * 64 * 64 * 64; size_t head_size = sbo_meta_size(data_size, 256 * 1024); - // meta 和 data 分离 uint8_t *meta = (uint8_t *)malloc(head_size); uint8_t *data = (uint8_t *)malloc(data_size); if (!meta || !data) { printf("malloc fail\n"); return 1; } diff --git a/tutorial/03_stress.c b/tutorial/03_stress.c index 0ad7251..8d59fd7 100644 --- a/tutorial/03_stress.c +++ b/tutorial/03_stress.c @@ -5,29 +5,27 @@ #include -#define SMALL_OBJ_SIZE 8 // 小对象大小 <=8byte +#define SMALL_OBJ_SIZE 8 + +#define DATA_SIZE (8ULL * 64 * 64 * 64) int main() { - const size_t data_size = 8ULL * 64 * 64 * 64 * 64; - // 初始化 slotsboxmalloc 的元数据和数据区，使用更大的内存池以支持高负载 - uint8_t *buddy = malloc(4 * 1024 * 1024); // 元数据区，增大到4MB - uint8_t *data = malloc(data_size); + uint8_t *buddy = calloc(1, 4 * 1024 * 1024); + uint8_t *data = malloc(DATA_SIZE); if (!buddy || !data) { perror("Failed to allocate memory for slotsboxmalloc"); return 1; } - // 初始化 slotsboxmalloc - if (sbo_init(buddy, 4 * 1024 * 1024, data_size) != 0) { + if (sbo_init(buddy, 4 * 1024 * 1024, DATA_SIZE) != 0) { printf("Failed to initialize slotsboxmalloc\n"); free(buddy); free(data); return 1; } - // 第一阶段：分配小对象直到写满 printf("Phase 1: Allocating small objects until full...\n"); - int capacity = 1000000; // 初始容量，动态扩展 + int capacity = 1000000; void **ptrs = malloc(capacity * sizeof(void *)); if (!ptrs) { perror("Failed to allocate ptrs array"); @@ -40,7 +38,7 @@ int main() { while (1) { uint64_t obj_offset = sbo_alloc(buddy, SMALL_OBJ_SIZE); if (obj_offset == (uint64_t)-1) { - break; // 分配失败，box已满 + break; } if (alloc_count >= capacity) { capacity *= 2; @@ -53,29 +51,25 @@ int main() { } } ptrs[alloc_count] = data + obj_offset; - *(uint64_t *)ptrs[alloc_count] = alloc_count; // 写入数据 + *(uint64_t *)ptrs[alloc_count] = alloc_count; alloc_count++; } printf("Allocated %d small objects\n", alloc_count); - // 第二阶段：无限循环随机free和malloc printf("Phase 2: Starting random free/malloc loop (2s limit)...\n"); - srand(time(NULL)); // 初始化随机种子 + srand(time(NULL)); long long loop_count = 0; int failed = 0; clock_t start = clock(); while (1) { if (loop_count > 0 && (loop_count % 5000 == 0) && (double)(clock() - start) / CLOCKS_PER_SEC > 2.0) break; - // 随机选择一个已分配的对象 int random_index = rand() % alloc_count; void *ptr_to_free = ptrs[random_index]; - // Free 随机对象 uint64_t offset = (uint8_t *)ptr_to_free - data; sbo_free(buddy, offset); - // 立即 Malloc 一个新的小对象 uint64_t new_offset = sbo_alloc(buddy, SMALL_OBJ_SIZE); if (new_offset == (uint64_t)-1) { printf("Re-allocation failed at loop %lld\n", loop_count); @@ -83,13 +77,11 @@ int main() { break; } - // 更新指针数组 ptrs[random_index] = data + new_offset; *(uint64_t *)ptrs[random_index] = random_index; loop_count++; - // 每10000次循环输出一次统计信息 if (loop_count % 10000 == 0) { clock_t current = clock(); double elapsed = (double)(current - start) / CLOCKS_PER_SEC; @@ -97,7 +89,6 @@ int main() { } } - // 清理 for (int i = 0; i < alloc_count; i++) { uint64_t offset= (uint8_t *)ptrs[i] - data; sbo_free(buddy, offset); diff --git a/tutorial/15_bench_dedicated.c b/tutorial/15_bench_dedicated.c index ee10598..48e4d87 100644 --- a/tutorial/15_bench_dedicated.c +++ b/tutorial/15_bench_dedicated.c @@ -11,14 +11,6 @@ #define META_SIZE (1024 * 1024) #define DATA_SIZE (8ULL * 64 * 64 * 64 * 64) -// Private allocator: each thread does alloc+free entirely within its home slot -// by directly calling box_alloc with slot-distinctive sizes that hash to -// different slots. Since box_alloc uses `size` to pick a starting slot, -// we pass thread_id as size to force slot_id = thread_id. -// -// This simulates N independent allocators sharing one meta area — the ideal -// concurrent access pattern with zero cache-line contention. - static const size_t kSz[] = {8, 64, 512, 4096}; static _Atomic long total = 0; diff --git a/tutorial/17_data_integrity.c b/tutorial/17_data_integrity.c index f92ea33..5553204 100644 --- a/tutorial/17_data_integrity.c +++ b/tutorial/17_data_integrity.c @@ -35,7 +35,6 @@ int main() { live_t live[MAX_LIVE]; int nlive = 0; - // 只选 8 的倍数，保证 tail 也对齐 size_t sizes[] = {8, 16, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384}; int ns = (int)(sizeof(sizes) / sizeof(sizes[0])); uint64_t seed = 1; diff --git a/tutorial/18_capacity.c b/tutorial/18_capacity.c deleted file mode 100644 index d0b7ebf..0000000 --- a/tutorial/18_capacity.c +++ /dev/null @@ -1,62 +0,0 @@ -#include -#include -#include -#include -#include - -static int exercise(void *meta, uint8_t *data, size_t data_size) { - uint64_t offsets[190]; - size_t sizes[190]; - int count = 0; - size_t size = data_size; - for (int depth = 0; depth < 3; depth++) { - size /= 64; - int n = depth == 2 ? 64 : 63; - for (int i = 0; i < n; i++) { - uint64_t off = sbo_alloc(meta, size); - if (off == UINT64_MAX || off + size > data_size || - sbo_allocated_size(meta, off) != size) - return 2; - for (int j = 0; j < count; j++) { - if (off < offsets[j] + sizes[j] && offsets[j] < off + size) - return 3; - } - offsets[count] = off; - sizes[count] = size; - data[off] = data[off + size - 1] = (uint8_t)count; - count++; - } - } - if (sbo_alloc(meta, size) != UINT64_MAX) - return 4; - - int hole = 143; - sbo_free(meta, offsets[hole]); - uint64_t replacement = sbo_alloc(meta, sizes[hole]); - if (replacement != offsets[hole]) - return 5; - for (int i = 0; i < count; i++) { - if (i != hole && (data[offsets[i]] != (uint8_t)i || - data[offsets[i] + sizes[i] - 1] != (uint8_t)i)) - return 6; - sbo_free(meta, offsets[i]); - } - return 0; -} - -int main(void) { - size_t data_size = 8ULL * 64 * 64 * 64 * 64; - size_t head_size = sbo_meta_size(data_size, 256 * 1024); - void *meta = malloc(head_size); - uint8_t *data = calloc(1, data_size); - int rc = 1; - if (meta) - memset(meta, 0xa5, head_size); - if (meta && data && sbo_init(meta, head_size, data_size) == 0) - rc = exercise(meta, data, data_size); - free(data); - free(meta); - if (rc == 0) - puts("capacity and reuse passed"); - return rc; -} diff --git a/tutorial/18_model_check.c b/tutorial/18_model_check.c new file mode 100644 index 0000000..92b60c4 --- /dev/null +++ b/tutorial/18_model_check.c @@ -0,0 +1,265 @@ +#define _POSIX_C_SOURCE 200809L +/* Randomized model check: no overlap, exact capacity, no false OOM. */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define META_SIZE (16 * 1024 * 1024) +#define DATA_SIZE (8ULL * 64 * 64 * 64 * 64) /* 128MB */ +#define UNITS (DATA_SIZE / 8) +#define FAIL64 ((uint64_t)-1) +#define LIVE 1024 + +static const size_t SZS[] = {1, 5, 8, 9, 16, 64, 100, 512, 1000, 4096, 8192, 16384, 65536, 200000, 1000000}; +#define NSZS (sizeof(SZS) / sizeof(SZS[0])) + +static uint8_t *g_meta, *g_data; +static _Atomic uint8_t *g_claim; +static _Atomic long g_err, g_phaseb_fail, g_alloc_ok; + +static uint64_t rng(uint64_t *s) { + uint64_t x = *s; x ^= x >> 12; x ^= x << 25; x ^= x >> 27; *s = x; + return x * 2685821657736338717ULL; +} + +/* canonical capacity: 8 * 64^(l-1) * m with m in [1,63] */ +static uint64_t expect_cap(size_t sz) { + uint64_t n = (sz + 7) / 8; + if (n <= 63) return n * 8; + uint64_t base = 1; + while (63 * base < n) base *= 64; + return ((n + base - 1) / base) * base * 8; +} + +typedef struct { uint64_t off, cap, tag; } ent_t; + +static void verify_and_free(ent_t *e, uint8_t owner) { + if (sbo_allocated_size(g_meta, e->off) != e->cap) g_err++; + uint64_t u0 = e->off / 8, un = e->cap / 8; + for (uint64_t u = u0; u < u0 + un; u++) + if (atomic_load_explicit(&g_claim[u], memory_order_relaxed) != owner) { g_err++; return; } + uint64_t v0, v1; + memcpy(&v0, g_data + e->off, 8); + memcpy(&v1, g_data + e->off + e->cap - 8, 8); + if (v0 != e->tag || v1 != e->tag) g_err++; + for (uint64_t u = u0; u < u0 + un; u++) + atomic_store_explicit(&g_claim[u], 0, memory_order_relaxed); + sbo_free(g_meta, e->off); +} + +typedef struct { int tid, iters; } targ_t; + +static void *churn_worker(void *a) { + targ_t *t = (targ_t *)a; + uint8_t owner = (uint8_t)(t->tid + 1); + uint64_t rs = 0x1234567ULL + (uint64_t)t->tid * 0x9e3779b9ULL; + ent_t *live = calloc(LIVE, sizeof(ent_t)); + if (!live) { g_err++; return NULL; } + long ok = 0; + int nl = 0; + for (int i = 0; i < t->iters; i++) { + if (nl > 0 && (nl == LIVE || (rng(&rs) & 1))) { + int idx = (int)(rng(&rs) % (uint64_t)nl); + verify_and_free(&live[idx], owner); + live[idx] = live[--nl]; + continue; + } + size_t req = SZS[rng(&rs) % NSZS]; + uint64_t off = sbo_alloc(g_meta, req); + if (off == FAIL64) continue; + ok++; + uint64_t cap = sbo_allocated_size(g_meta, off); + uint64_t exp = expect_cap(req); + if (cap != exp || off + cap > DATA_SIZE || (off & 7)) { + printf("BAD cap req=%zu off=%lu cap=%lu exp=%lu\n", + req, (unsigned long)off, (unsigned long)cap, (unsigned long)exp); + g_err++; sbo_free(g_meta, off); continue; + } + uint64_t u0 = off / 8, un = cap / 8; + for (uint64_t u = u0; u < u0 + un; u++) { + uint8_t exp = 0; + if (!atomic_compare_exchange_strong_explicit(&g_claim[u], &exp, owner, + memory_order_relaxed, memory_order_relaxed)) { + printf("OVERLAP unit=%lu off=%lu held=%d\n", (unsigned long)u, (unsigned long)off, exp); + g_err++; + } + } + uint64_t tag = rng(&rs); + memcpy(g_data + off, &tag, 8); + memcpy(g_data + off + cap - 8, &tag, 8); + live[nl++] = (ent_t){off, cap, tag}; + } + for (int i = 0; i < nl; i++) verify_and_free(&live[i], owner); + free(live); + g_alloc_ok += ok; + return NULL; +} + +static void *allocfree_worker(void *a) { + targ_t *t = (targ_t *)a; + static const size_t szs[] = {8, 64, 512, 4096}; + long fail = 0; + for (int i = 0; i < t->iters; i++) { + uint64_t off = sbo_alloc(g_meta, szs[i & 3]); + if (off == FAIL64) { fail++; continue; } + sbo_free(g_meta, off); + } + g_phaseb_fail += fail; + return NULL; +} + +static void spawn_threads(void *(*fn)(void *), int nthr, int iters) { + pthread_t *tids = malloc((size_t)nthr * sizeof(pthread_t)); + targ_t *args = malloc((size_t)nthr * sizeof(targ_t)); + if (!tids || !args) { g_err++; free(tids); free(args); return; } + int created = 0; + for (int i = 0; i < nthr; i++) { + args[i] = (targ_t){i, iters}; + if (pthread_create(&tids[i], NULL, fn, &args[i]) != 0) { g_err++; break; } + created++; + } + for (int i = 0; i < created; i++) + if (pthread_join(tids[i], NULL) != 0) g_err++; + free(tids); free(args); +} + +static void preflight(void) { + for (size_t i = 0; i < NSZS; i++) { + uint64_t off = sbo_alloc(g_meta, SZS[i]); + if (off == FAIL64) { printf("preflight alloc %zu failed\n", SZS[i]); g_err++; continue; } + if (sbo_allocated_size(g_meta, off) != expect_cap(SZS[i])) g_err++; + sbo_free(g_meta, off); + } +} + +static void run_phase(const char *name, void *(*fn)(void *), int nthr, int iters) { + memset(g_meta, 0, META_SIZE); + if (sbo_init(g_meta, META_SIZE, DATA_SIZE) != 0) { printf("%s: init fail\n", name); g_err++; return; } + if (fn == churn_worker) preflight(); + for (uint64_t u = 0; u < UNITS; u++) + atomic_store_explicit(&g_claim[u], 0, memory_order_relaxed); + g_alloc_ok = 0; + spawn_threads(fn, nthr, iters); + if (fn == churn_worker && g_alloc_ok == 0) { + printf("%s: no successful allocations\n", name); + g_err++; + } + long leaked = 0; + for (uint64_t u = 0; u < UNITS; u++) + if (atomic_load_explicit(&g_claim[u], memory_order_relaxed)) leaked++; + if (leaked) { printf("%s: leaked_units=%ld\n", name, leaked); g_err++; } + printf("%s done (errors so far=%ld)\n", name, (long)g_err); +} + +static void phase_edges(void) { + memset(g_meta, 0, META_SIZE); + if (sbo_init(g_meta, META_SIZE, DATA_SIZE) != 0) { printf("edges: init fail\n"); g_err++; return; } + const size_t bad[] = {0, SIZE_MAX, SIZE_MAX - 3, 2064385}; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) + if (sbo_alloc(g_meta, bad[i]) != FAIL64) { printf("edges: alloc %zu must fail\n", bad[i]); g_err++; } + uint64_t off = sbo_alloc(g_meta, 2064384); + if (off == FAIL64 || sbo_allocated_size(g_meta, off) != 2064384) g_err++; + if (off != FAIL64) sbo_free(g_meta, off); + off = sbo_alloc(g_meta, 512); + if (off == FAIL64) { g_err++; return; } + if (sbo_allocated_size(g_meta, off + 8) != 0) g_err++; + sbo_free(g_meta, off + 8); + if (sbo_allocated_size(g_meta, off) != 512) g_err++; + sbo_free(g_meta, off); + off = sbo_alloc(g_meta, 8); + if (off == FAIL64) { g_err++; return; } + sbo_free(g_meta, off + 3); + if (sbo_allocated_size(g_meta, off) != 8) g_err++; + sbo_free(g_meta, off); + if (sbo_allocated_size(g_meta, off) != 0) g_err++; + sbo_free(g_meta, off); + g_meta[14] = '2'; + if (sbo_alloc(g_meta, 8) != FAIL64) g_err++; + if (sbo_allocated_size(g_meta, off) != 0) g_err++; + sbo_free(g_meta, off); + uint8_t *tiny = calloc(1, 8); + if (!tiny || sbo_init(tiny, 8, DATA_SIZE) == 0) g_err++; + free(tiny); + printf("D edges done (errors so far=%ld)\n", (long)g_err); +} + +static void phase_small_arenas(void) { + static const size_t datas[] = {512, 32768, 2097152}; + for (size_t d = 0; d < sizeof(datas) / sizeof(datas[0]); d++) { + size_t ms = sbo_meta_size(datas[d], 256 * 1024); + uint8_t *m = calloc(1, ms); + if (!m) { g_err++; continue; } + if (sbo_init(m, ms, datas[d]) != 0) { printf("small %zu: init fail\n", datas[d]); g_err++; free(m); continue; } + size_t maxsz = datas[d] / 64 * 63; + uint64_t o1 = sbo_alloc(m, 504), o2 = (datas[d] > 512) ? sbo_alloc(m, 504) : 0; + if (o1 == FAIL64 || o2 == FAIL64) g_err++; + if (o1 != FAIL64) sbo_free(m, o1); + if (datas[d] > 512 && o2 != FAIL64) sbo_free(m, o2); + uint64_t off = sbo_alloc(m, maxsz); + if (off == FAIL64 || sbo_allocated_size(m, off) != expect_cap(maxsz)) g_err++; + if (off != FAIL64) sbo_free(m, off); + free(m); + } + uint8_t *m2 = calloc(1, 1136); + if (!m2 || sbo_init(m2, 1136, 2097152) != 0) { g_err++; free(m2); return; } + if (sbo_alloc(m2, 8) != FAIL64) g_err++; + uint64_t b1 = sbo_alloc(m2, 1048576), b2 = sbo_alloc(m2, 1048576); + if (b1 == FAIL64 || b2 == FAIL64) g_err++; + if (b1 != FAIL64) sbo_free(m2, b1); + if (b2 != FAIL64) sbo_free(m2, b2); + free(m2); + uint8_t *m = calloc(1, 1 << 20); + if (!m) { g_err++; return; } + if (sbo_init(m, 1 << 20, 1000) == 0) g_err++; + memset(m, 0, 1 << 20); + if (sbo_init(m, 1 << 20, 8) == 0) g_err++; + free(m); + printf("E small arenas done (errors so far=%ld)\n", (long)g_err); +} + +static void phase_single_shard_mt(int nthr, int iters) { + size_t ms = sbo_meta_size(2097152, 256 * 1024); + uint8_t *saved = g_meta; + g_meta = calloc(1, ms); + if (!g_meta || sbo_init(g_meta, ms, 2097152) != 0) { + printf("single-shard: init fail\n"); + g_err++; + free(g_meta); + g_meta = saved; + return; + } + g_phaseb_fail = 0; + spawn_threads(allocfree_worker, nthr, iters); + if (g_phaseb_fail) { printf("single-shard: %ld false failures\n", (long)g_phaseb_fail); g_err++; } + free(g_meta); + g_meta = saved; + printf("F single-shard MT done (errors so far=%ld)\n", (long)g_err); +} + +int main(int argc, char **argv) { + if (argc > 3) { printf("usage: %s [threads 1..254] [iters>=1]\n", argv[0]); return 2; } + int nthr = argc > 1 ? atoi(argv[1]) : 4; + int iters = argc > 2 ? atoi(argv[2]) : 50000; + if (nthr < 1 || nthr > 254 || iters < 1) { + printf("usage: %s [threads 1..254] [iters>=1]\n", argv[0]); + return 2; + } + g_meta = malloc(META_SIZE); g_data = malloc(DATA_SIZE); + g_claim = calloc(UNITS, sizeof(_Atomic uint8_t)); + if (!g_meta || !g_data || !g_claim) { printf("out of memory\n"); return 2; } + run_phase("A churn ST", churn_worker, 1, iters); + run_phase("B allocfree MT (no false OOM)", allocfree_worker, nthr, iters); + if (g_phaseb_fail) { printf("B: %ld allocations falsely failed under contention\n", (long)g_phaseb_fail); g_err++; } + run_phase("C churn MT", churn_worker, nthr, iters); + phase_edges(); + phase_small_arenas(); + phase_single_shard_mt(nthr, iters); + printf("18_model_check: %s\n", g_err == 0 ? "PASS" : "FAIL"); + free(g_meta); free(g_data); free(g_claim); + return g_err == 0 ? 0 : 1; +} diff --git a/tutorial/19_bitmap_oracle.c b/tutorial/19_bitmap_oracle.c deleted file mode 100644 index fadcbb8..0000000 --- a/tutorial/19_bitmap_oracle.c +++ /dev/null @@ -1,154 +0,0 @@ -#include -#include -#include -#include -#include -#include - -static void *meta; -static sbo_box_t *root; -static uint64_t cases; - -static uint8_t longest(uint64_t bits) { - uint8_t best = 0, run = 0; - for (int i = 0; i < 64; i++) { - run = bits & (UINT64_C(1) << i) ? (uint8_t)(run + 1) : 0; - if (run > best) best = run; - } - return best; -} - -static int rightmost(uint64_t bits, int boundary, int count) { - for (int end = 63; end >= boundary; end--) { - int start = end - count + 1; - if (start < boundary) break; - int free_run = 1; - for (int i = start; i <= end; i++) { - if (!(bits & (UINT64_C(1) << i))) { free_run = 0; break; } - } - if (free_run) return start; - } - return -1; -} - -static void fail(uint64_t bits, int boundary, int count, const char *what) { - fprintf(stderr, "%s: bits=%016" PRIx64 " boundary=%d count=%d\n", - what, bits, boundary, count); - exit(1); -} - -static void check_with_hint(uint64_t bits, int boundary, int count, sbo_usage_t hint) { - root->box_boundary = (int16_t)boundary; - root->obj_boundary = 63; - root->max_obj_cap = longest(bits); - root->child_max_cap = hint; - for (int i = 0; i < 8; i++) root->free_bitmap[i] = (uint8_t)(bits >> (8 * i)); - for (int i = 0; i < 64; i++) - root->slots[i].state = bits & (UINT64_C(1) << i) ? SBO_FREE : SBO_OBJ_START; - - sbo_box_t expected = *root; - int start = rightmost(bits, boundary, count); - uint64_t offset = sbo_alloc(meta, (size_t)count * 8); - uint64_t want = start < 0 ? UINT64_MAX : (uint64_t)start * 8; - if (offset != want) fail(bits, boundary, count, "rightmost placement"); - if (start >= 0) { - uint64_t mask = (UINT64_MAX >> (64 - count)) << start; - for (int i = start; i < start + count; i++) { - expected.free_bitmap[i >> 3] &= (uint8_t)~(1U << (i & 7)); - expected.slots[i].state = i == start ? SBO_OBJ_START : SBO_OBJ_CONT; - } - expected.obj_boundary = (int16_t)(start - 1); - expected.max_obj_cap = longest(bits & ~mask); - if (sbo_allocated_size(meta, offset) != (uint64_t)count * 8) - fail(bits, boundary, count, "allocated size"); - } - if (memcmp(root, &expected, sizeof expected)) fail(bits, boundary, count, "allocation state"); - if (start >= 0) { - sbo_free(meta, offset); - for (int i = start; i < start + count; i++) { - expected.free_bitmap[i >> 3] |= (uint8_t)(1U << (i & 7)); - expected.slots[i].state = SBO_FREE; - } - expected.max_obj_cap = longest(bits); - if (memcmp(root, &expected, sizeof expected)) fail(bits, boundary, count, "free state"); - } - cases++; -} - -static void all_counts(uint64_t bits) { - sbo_usage_t zero_hint = {0, 0}; - for (int boundary = 0; boundary <= 64; boundary++) - for (int count = 1; count <= 63; count++) - check_with_hint(bits, boundary, count, zero_hint); -} - -static void check_multilevel_propagation(void) { - /* 2-level pool (32KB = 8 * 64^2): verify depth-1 child boxes (parent == 0) - * propagate capacity changes to root and seamlessly spill to sibling boxes. */ - size_t data_size = 8ULL * 64 * 64; - size_t head_size = sbo_meta_size(data_size, 64 * 1024); - void *m2 = calloc(1, head_size); - if (!m2 || sbo_init(m2, head_size, data_size) != 0) { - fprintf(stderr, "multilevel init failed\n"); - exit(1); - } - uint64_t offs[128]; - for (int i = 0; i < 128; i++) { - offs[i] = sbo_alloc(m2, 8); - if (offs[i] == UINT64_MAX || sbo_allocated_size(m2, offs[i]) != 8) { - fprintf(stderr, "multilevel alloc failed at %d\n", i); - exit(1); - } - } - for (int i = 0; i < 128; i += 2) sbo_free(m2, offs[i]); - for (int i = 0; i < 128; i += 2) { - uint64_t re = sbo_alloc(m2, 8); - if (re == UINT64_MAX) { - fprintf(stderr, "multilevel realloc failed at %d\n", i); - exit(1); - } - offs[i] = re; - } - for (int i = 0; i < 128; i++) sbo_free(m2, offs[i]); - free(m2); -} - -int main(void) { - meta = calloc(1, 4096); - if (!meta || sbo_init(meta, 4096, 512) != 0) return 1; - sbo_meta_t *m = meta; - root = (sbo_box_t *)((uint8_t *)(m + 1) - + m->root_slots * (sizeof(blocks_meta_t) + sizeof(sbo_lock_t)) - + m->sizeof_block_head); - - /* Exercise leaf capacity guard even when subtree hint is optimistic. */ - sbo_usage_t opt_hint = {2, 1}; - for (uint64_t bits = 0; bits < 256; bits++) { - for (int shift = 0; shift <= 56; shift += 28) - for (int boundary = 0; boundary <= 64; boundary++) - for (int count = 1; count <= 9; count++) - check_with_hint(bits << shift, boundary, count, opt_hint); - } - all_counts(0); - all_counts(UINT64_MAX); - all_counts(UINT64_C(0xAAAAAAAAAAAAAAAA)); - all_counts(UINT64_C(0x5555555555555555)); - for (int bit = 0; bit < 64; bit++) { - all_counts(UINT64_C(1) << bit); - all_counts(~(UINT64_C(1) << bit)); - } - uint64_t seed = UINT64_C(0x937bf272d9ea1563); - for (int i = 0; i < 64; i++) { - seed ^= seed << 13; seed ^= seed >> 7; seed ^= seed << 17; - uint64_t bits = seed; - if (i % 3 == 0) bits &= UINT64_C(0x5555555555555555); - else if (i % 3 == 1) bits |= UINT64_C(0xAAAAAAAAAAAAAAAA); - all_counts(bits); - } - free(meta); - - check_multilevel_propagation(); - - printf("PASS bitmap regression: %" PRIu64 " cases\n", cases); - return 0; -} diff --git a/tutorial/CMakeLists.txt b/tutorial/CMakeLists.txt index 6fbd166..15b0ea9 100644 --- a/tutorial/CMakeLists.txt +++ b/tutorial/CMakeLists.txt @@ -73,11 +73,8 @@ add_executable(17_data_integrity 17_data_integrity.c) target_link_libraries(17_data_integrity slotsboxmalloc) add_test(NAME 17_data_integrity COMMAND 17_data_integrity) -add_executable(18_capacity 18_capacity.c) -target_link_libraries(18_capacity slotsboxmalloc) -add_test(NAME 18_capacity COMMAND 18_capacity) - -add_executable(19_bitmap_oracle 19_bitmap_oracle.c) -target_link_libraries(19_bitmap_oracle slotsboxmalloc) -add_test(NAME 19_bitmap_oracle COMMAND 19_bitmap_oracle) - +# -- 18: randomized model check (overlap / capacity / no false OOM) -- +add_executable(18_model_check 18_model_check.c) +target_link_libraries(18_model_check slotsboxmalloc Threads::Threads) +add_test(NAME 18_model_check COMMAND 18_model_check 4 10000) +set_tests_properties(18_model_check PROPERTIES TIMEOUT 120) diff --git a/tutorial/test.py b/tutorial/test.py index 14d312d..da831ab 100755 --- a/tutorial/test.py +++ b/tutorial/test.py @@ -9,7 +9,7 @@ RED, GREEN, YELLOW, NC = "\033[0;31m", "\033[0;32m", "\033[1;33m", "\033[0m" ROOT = Path(__file__).resolve().parent.parent FAIL_CSV = (ROOT / "tutorial" / "test_failures.csv").resolve() -TIMEOUT = 60 # 03_stress 有无界循环，给较长超时 +TIMEOUT = 60 def discover(build_dir: Path) -> list[Path]: