diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3def7c0..7c669e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,3 +7,5 @@ jobs: - uses: actions/checkout@v4 - name: 构建 run: cmake -B build && cmake --build build + - name: Run unit tests + run: ctest --test-dir build --output-on-failure diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3183d48..16af9ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,10 +13,10 @@ jobs: - uses: actions/checkout@v4 - name: 构建 run: cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build - - name: 构建 CLI(cli/ 独立 Cargo 项目,走 dispatch 前端 C ABI) + - name: Verify CLI RUNPATH (built by cmake kvspace-cli target) run: | - # CLI 用 #[link(name = "kvspace")] 链本仓刚构建的前端 → 把 build/ 交给链接器 - RUSTFLAGS="-L native=$PWD/build" cargo build --release --manifest-path cli/Cargo.toml + set -euo pipefail + readelf -d cli/target/release/kvspace | grep -F 'RUNPATH' | grep -F '$ORIGIN/../lib' - name: 打包 ABI 产物 run: | set -euo pipefail diff --git a/CMakeLists.txt b/CMakeLists.txt index b64e85c..350c0ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,14 +56,22 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/kvspace.pc DESTINATION ${CMAKE_INSTALL # ── CLI(cli/ 独立 Cargo 项目)────────────────────────────────────── # 走本仓 dispatch 前端的 C ABI,故须在 kvspace 目标之后构建; # 安装到 /bin(prefix 默认 /usr → /usr/bin,与 kvlang 的 CLI 一致)。 -# RUNPATH 用相对自身的 $ORIGIN/../lib:可执行在 bin/、前端在 lib/,tarball -# 换机器后仍然解析。$ 转义成 $$ 以免被 CMake 当变量展开。 +# Embed relative RUNPATH entries: +# $ORIGIN/../lib -- release tarball (bin/ + lib/) +# $ORIGIN/..//kvspace -- cmake --install (//kvspace) +# Use @loader_path on macOS; escape $ as $$ for CMake. +# release.yml packages this binary directly without rebuilding. find_program(CARGO_EXECUTABLE cargo) if(CARGO_EXECUTABLE) set(KVSPACE_CLI_BIN "${CMAKE_CURRENT_SOURCE_DIR}/cli/target/release/kvspace") + if(APPLE) + set(KVSPACE_CLI_ORIGIN "@loader_path") + else() + set(KVSPACE_CLI_ORIGIN "$$ORIGIN") + endif() add_custom_target(kvspace-cli ALL COMMAND ${CMAKE_COMMAND} -E env - "RUSTFLAGS=-L native=$ -C link-arg=-Wl,-rpath,'$$ORIGIN/../lib'" + "RUSTFLAGS=-L native=$ -C link-arg=-Wl,-rpath,'${KVSPACE_CLI_ORIGIN}/../lib' -C link-arg=-Wl,-rpath,'${KVSPACE_CLI_ORIGIN}/../${CMAKE_INSTALL_LIBDIR}/kvspace'" "${CARGO_EXECUTABLE}" build --release --manifest-path "${CMAKE_CURRENT_SOURCE_DIR}/cli/Cargo.toml" BYPRODUCTS "${KVSPACE_CLI_BIN}" diff --git a/README.md b/README.md index ca4d296..7a4a2ae 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ KVSpace 的 dispatch 前端。消费者只链接 `libkvspace`,运行期按 DSN XValue 构造(`kvspaceNew*`/`kvspaceXvalueHeadNew*`)无 handle,由前端静态实现, head 格式 byte-identical,两个后端与前端三者共用同一份契约。 -后端一律按固定目录查找(Linux `/usr/lib/kvspace`,macOS `/usr/local/lib/kvspace`); +Backend lookup order: frontend `.so` directory, its `kvspace/` subdirectory, `../kvspace/`, +falling back to system paths (Linux `/usr/lib/kvspace`, macOS `/usr/local/lib/kvspace`); 若宿主已按 SONAME 加载过后端(单二进制 `memfd` 场景),前端用 `dlopen(soname, RTLD_NOW | RTLD_NOLOAD)` 直接复用该对象,不重复映射、不 dlclose。 运行期没有任何目录/后端类环境变量——`KVSPACE` 是唯一的运行时变量(DSN 选后端)。 diff --git a/cli/kvspace.rs b/cli/kvspace.rs index 57faa0d..a939abb 100644 --- a/cli/kvspace.rs +++ b/cli/kvspace.rs @@ -8,34 +8,9 @@ use std::process::exit; /* 固定 head 8 字节(见 kvspace/xvalue_head.h):body 偏移恒 8。 */ const KVSPACE_XVALUE_HEAD_LEN: usize = 8; const KVSPACE_XVALUE_ST_SHORT: u8 = 0; - -/* 与 kvspace.h 的 kvspaceHead_t 布局一致(ro/vid 已从 ABI 删除)。 */ -#[repr(C)] -struct Head { - headlen: u16, - r#ref: u8, - storetype: u8, - body_len: i32, - ndim: i32, - dims: [i32; 8], - langtype: [u8; 256], - langtype_len: i32, - body_offset: i32, - body_cap: u64, -} - -fn zero_head() -> Head { - unsafe { std::mem::zeroed() } -} - -fn langtype_bytes(h: &Head) -> &[u8] { - let n = h - .langtype - .iter() - .position(|&b| b == 0) - .unwrap_or(h.langtype.len()); - &h.langtype[..n] -} +/* head.ptr: 0=inline, 1=ptr, 2=@ext (KVSPACE_XVALUE_PTR_*). */ +const KVSPACE_XVALUE_PTR_PTR: u8 = 1; +const KVSPACE_XVALUE_PTR_EXT: u8 = 2; #[link(name = "kvspace")] extern "C" { @@ -163,7 +138,6 @@ extern "C" { out: *mut *mut u8, out_len: *mut u64, ) -> c_int; - fn kvspaceGetHead(h: *mut c_void, key: *const c_char, out: *mut Head) -> c_int; } fn cs(s: &str) -> *const c_char { @@ -191,9 +165,10 @@ fn fail_op(err: &[u8; 256], op: &str) -> ! { } struct Value { - kind: String, - r#ref: i32, - dims: Vec, + kind: String, // Base kind (langtype without leading [dims]) + r#ref: u8, // head.ptr: 0=inline, 1=ptr, 2=@ext + storetype: u8, // head.storetype + dims: Vec, // Leading [dims] parsed from langtype body: Vec, } @@ -202,34 +177,25 @@ struct Value { 值自描述:直接解 wire,不经 kvspaceGetHead(它对 Ptr 穿透,读不到本体)。 */ fn from_wire(data: &[u8]) -> Value { if data.len() < KVSPACE_XVALUE_HEAD_LEN { - return Value { kind: String::new(), r#ref: 0, dims: vec![], body: vec![] }; + return Value { + kind: String::new(), + r#ref: 0, + storetype: 0, + dims: vec![], + body: vec![], + }; } let body_len = wire_len(data) as usize; - let storetype = data[6]; - let ptr = data[7]; - let r#ref = match ptr { - 1 => 1, - 2 => 2, - _ => { - let _ = storetype; - 0 - } - }; - let lt = wire_langtype(data, body_len); - let (_ltdims, kind) = parse_langtype(<); + let (dims, kind) = parse_langtype(&wire_langtype(data, body_len)); /* body 只到 head.len 为止;其后是 attr 区,不属 body。 */ let start = KVSPACE_XVALUE_HEAD_LEN; - let end = (start + body_len).min(data.len()); - let body = if end > start { - data[start..end].to_vec() - } else { - vec![] - }; + let end = start.saturating_add(body_len).min(data.len()); Value { kind, - r#ref, - dims: vec![], - body, + r#ref: data[7], + storetype: data[6], + dims, + body: data[start..end].to_vec(), } } @@ -244,8 +210,8 @@ fn wire_len(data: &[u8]) -> u64 { /// 扫 attr 区取 tag=1(langtype);无记录/坏 attr → 空串。 fn wire_langtype(data: &[u8], body_len: usize) -> String { - let base = KVSPACE_XVALUE_HEAD_LEN as usize + body_len; - if base + 2 > data.len() { + let base = KVSPACE_XVALUE_HEAD_LEN.saturating_add(body_len); + if base.saturating_add(2) > data.len() { return String::new(); } let alen = (data[base] as usize) | ((data[base + 1] as usize) << 8); @@ -286,15 +252,6 @@ fn parse_langtype(lt: &str) -> (Vec, String) { (Vec::new(), lt.to_string()) } -fn le_u32(b: &[u8]) -> u32 { - (b[0] as u32) | ((b[1] as u32) << 8) | ((b[2] as u32) << 16) | ((b[3] as u32) << 24) -} -fn le_u64(b: &[u8]) -> u64 { - b.iter() - .enumerate() - .fold(0u64, |a, (i, &x)| a | ((x as u64) << (8 * i))) -} - fn fmt_float(v: f64) -> String { let s = format!("{}", v); if s.contains('.') { @@ -304,54 +261,89 @@ fn fmt_float(v: f64) -> String { } } +/// Element width in bytes for fixed-width scalars; None otherwise. +fn scalar_width(kind: &str) -> Option { + match kind { + "bool" | "int8" | "uint8" => Some(1), + "int16" | "uint16" => Some(2), + "int32" | "uint32" | "float32" => Some(4), + "int64" | "uint64" | "float64" => Some(8), + _ => None, + } +} + +/// Decode one little-endian fixed-width scalar; returns None on unknown kind or short slice. +fn scalar_text(kind: &str, b: &[u8]) -> Option { + let w = scalar_width(kind)?; + let b = b.get(..w)?; + Some(match kind { + "bool" => (b[0] != 0).to_string(), + "int8" => (b[0] as i8).to_string(), + "uint8" => b[0].to_string(), + "int16" => i16::from_le_bytes(b.try_into().ok()?).to_string(), + "uint16" => u16::from_le_bytes(b.try_into().ok()?).to_string(), + "int32" => i32::from_le_bytes(b.try_into().ok()?).to_string(), + "uint32" => u32::from_le_bytes(b.try_into().ok()?).to_string(), + "float32" => fmt_float(f32::from_le_bytes(b.try_into().ok()?) as f64), + "int64" => i64::from_le_bytes(b.try_into().ok()?).to_string(), + "uint64" => u64::from_le_bytes(b.try_into().ok()?).to_string(), + "float64" => fmt_float(f64::from_le_bytes(b.try_into().ok()?)), + _ => return None, + }) +} + +fn join_dims(dims: &[i32]) -> String { + dims.iter() + .map(|d| d.to_string()) + .collect::>() + .join(",") +} + fn plain(v: &Value) -> String { // ref 先于 kind 判空:不约束指针(空 kindexpr)也是合法值。 - if v.r#ref == 1 { + if v.r#ref == KVSPACE_XVALUE_PTR_PTR { return format!("→{}", String::from_utf8_lossy(&v.body)); } + if v.r#ref == KVSPACE_XVALUE_PTR_EXT { + return format!("@{}", String::from_utf8_lossy(&v.body)); + } if v.kind.is_empty() { return "None".to_string(); } + if let Some(w) = scalar_width(&v.kind) { + // Format fixed-width numeric tensors ([dims]) element-by-element. + if !v.dims.is_empty() { + let items: Vec = v + .body + .chunks_exact(w) + .filter_map(|c| scalar_text(&v.kind, c)) + .collect(); + return format!("[{}]", items.join(",")); + } + return scalar_text(&v.kind, &v.body) + .unwrap_or_else(|| format!("", v.body.len())); + } match v.kind.as_str() { - "bool" => (v.body.first().copied().unwrap_or(0) != 0).to_string(), - "int8" => (v.body.first().map(|&b| b as i8).unwrap_or(0) as i64).to_string(), - "int16" => (i16::from_le_bytes([v.body[0], v.body[1]]) as i64).to_string(), - "int32" => { - (i32::from_le_bytes([v.body[0], v.body[1], v.body[2], v.body[3]]) as i64).to_string() - } - "int64" => (i64::from_le_bytes(v.body[..8].try_into().unwrap())).to_string(), - "uint8" => v.body.first().copied().unwrap_or(0).to_string(), - "uint16" => (u16::from_le_bytes([v.body[0], v.body[1]])).to_string(), - "uint32" => le_u32(&v.body).to_string(), - "uint64" => le_u64(&v.body).to_string(), - "float32" => fmt_float(f32::from_le_bytes(v.body[..4].try_into().unwrap()) as f64), - "float64" => fmt_float(f64::from_le_bytes(v.body[..8].try_into().unwrap())), - "char/utf8" | "char/ascii" => String::from_utf8_lossy(&v.body).into_owned(), "char/utf32" => v .body - .chunks(4) - .map(|c| char::from_u32(le_u32(c)).unwrap_or('\u{FFFD}')) + .chunks_exact(4) + .map(|c| { + char::from_u32(u32::from_le_bytes([c[0], c[1], c[2], c[3]])).unwrap_or('\u{FFFD}') + }) .collect(), - "stringkeymap" => format!( - "map[{}]", - v.dims - .iter() - .map(|d| d.to_string()) - .collect::>() - .join(",") - ), + "stringkeymap" => format!("map[{}]", join_dims(&v.dims)), "index" => format!("({})", v.dims.first().copied().unwrap_or(0)), - "extindex" => String::from_utf8_lossy(&v.body).into_owned(), + // char/utf8, char/ascii, and other kinds: decode as lossy UTF-8. _ => String::from_utf8_lossy(&v.body).into_owned(), } } fn format_value(v: &Value) -> String { - if v.r#ref == 1 { + if v.r#ref == KVSPACE_XVALUE_PTR_PTR { return format!("→{}:{}", String::from_utf8_lossy(&v.body), v.kind); } if v.kind.is_empty() { - return "None".to_string(); + return plain(v); // None or @ext locator } format!("{}:{}", v.kind, plain(v)) } @@ -368,13 +360,13 @@ fn get(kv: *mut c_void, key: &str) -> Vec { } } -// 解析 value 语法为 (wire, langtype):langtype 是无前缀裸 kindexpr(int64/[]char/utf8/键·值), -// 由 codec 写进值内 attr 区(tag=1 记录,随 head+body 单 key 整体落);空 = 无类型。 -fn parse_value(raw: &str) -> (Vec, String) { +// Parse CLI value syntax into full XValue wire bytes (langtype stored in tag=1 attr). +// Empty Vec represents `nil:` (delete). +fn parse_value(raw: &str) -> Vec { let mut out: *mut u8 = std::ptr::null_mut(); let mut len: u32 = 0; if let Some(rest) = raw.strip_prefix('*') { - // `*:kindexpr:target`(kindexpr 在前,与 string:/int: 同构);`*:target` 为不约束指针。 + // `*:kindexpr:target` (typed ptr) or `*:target` (unconstrained ptr). let rest = rest.strip_prefix(':').unwrap_or(rest); let (k, t) = match rest.find(':') { Some(i) => (&rest[..i], &rest[i + 1..]), @@ -383,14 +375,14 @@ fn parse_value(raw: &str) -> (Vec, String) { if unsafe { kvspaceNewPtr(cs(k), cs(t), &mut out, &mut len) != 0 } { fatalf(&format!("parse value failed: {:?}", raw)); } - let v = take_wire(out, len); - return (v, k.to_string()); // kindexpr 无 * 前缀;不约束(空 kindexpr)→ 空 langtype + // Unprefixed kindexpr; empty kindexpr -> unconstrained (empty langtype). + return take_wire(out, len); } if let Some(langtype) = raw.strip_prefix("map:") { if !langtype.contains('·') { fatalf("map requires key·value langtype"); } - // 空 body 的短定长头(langtype 写进值内 attr 区)。 + // Empty-body short header with langtype in the attr region. let mut len64: u64 = 0; if unsafe { kvspaceXvalueHeadNewScalar( @@ -404,8 +396,7 @@ fn parse_value(raw: &str) -> (Vec, String) { } { fatalf(&format!("parse value failed: {:?}", raw)); } - let v = take_wire(out, len64 as u32); - return (v, langtype.to_string()); // 裸 kindexpr(如 int·string) + return take_wire(out, len64 as u32); // bare kindexpr (e.g. int·string) } let split = raw.find(':').unwrap_or(raw.len()); let kind = &raw[..split]; @@ -445,11 +436,12 @@ fn parse_value(raw: &str) -> (Vec, String) { .parse() .unwrap_or_else(|_| fatalf(&format!("parse value failed: {:?}", raw))); // 4 字节小端 f32 body 的短定长头。 + let le = f.to_le_bytes(); let mut len64: u64 = 0; if unsafe { kvspaceXvalueHeadNewScalar( KVSPACE_XVALUE_ST_SHORT, - &f as *const f32 as *const u8, + le.as_ptr(), 4, cs(langtype), &mut out, @@ -484,9 +476,9 @@ fn parse_value(raw: &str) -> (Vec, String) { fatalf(&format!("parse value failed: {:?}", raw)); } }, - "nil" => return (Vec::new(), String::new()), // 删除(无值) + "nil" => return Vec::new(), // delete (no value) "index" => fatalf("use mkindex for directories"), - // 无 kind 前缀或未识别前缀 → 默认按字符串处理(见 help 的 value 语法)。 + // Default to string when no recognized kind prefix is present. _ => unsafe { if kvspaceNewChar( raw.as_ptr(), @@ -500,7 +492,7 @@ fn parse_value(raw: &str) -> (Vec, String) { } }, } - (take_wire(out, len), langtype.to_string()) + take_wire(out, len) } // codec 产出为 frontend malloc,读出即 free;失败已 fatalf 退出,到这里的 out 恒非空。 @@ -512,12 +504,16 @@ fn take_wire(out: *mut u8, len: u32) -> Vec { } } -// 单 key 写整值(head + body + attr 尾,langtype 已在 attr 区内,见 #571)。 +// Single-key full-value write (head + body + attr tail). fn set_one(kv: *mut c_void, key: &str, val: &[u8]) { if val.is_empty() { + // `nil:` deletes the key; report errors just like `del` (missing key returns 0). let key_ptr = cs(key); let mut err = [0u8; 256]; - unsafe { kvspaceDel(kv, &key_ptr, 1, err.as_mut_ptr() as *mut c_char, 256) }; + let rc = unsafe { kvspaceDel(kv, &key_ptr, 1, err.as_mut_ptr() as *mut c_char, 256) }; + if rc != 0 { + fail_op(&err, &format!("set {} nil", key)); + } return; } let ck = cs(key); @@ -542,15 +538,19 @@ fn set_one(kv: *mut c_void, key: &str, val: &[u8]) { } } -// 前缀遍历:先 ListLen 定计数,再逐 idx ListAt 取名(借用回收缓冲,读出即自持)。resolve=1 穿透 link。 -// ListLen/ListAt 要求前缀是目录形(/ 或 · 结尾),裸前缀(如 /t)在此归一,避免 `list /t` 静默为空。 -fn list_names(kv: *mut c_void, prefix: &str, expand_ext: bool) -> Vec { - let dir = if prefix == "/" || prefix.ends_with('/') || prefix.ends_with('·') { +// Normalize directory prefix (trailing `/` or `·`): ListLen/ListAt, mkindex, and +// extindex mounts address directory keys; append `/` to bare prefixes like `/t`. +fn dir_form(prefix: &str) -> String { + if prefix.ends_with('/') || prefix.ends_with('·') { prefix.to_string() } else { format!("{}/", prefix) - }; - let p = cs(&dir); + } +} + +// Prefix listing via ListLen + ListAt (resolve=1 follows links; expand_ext=false lists own children only). +fn list_names(kv: *mut c_void, prefix: &str, expand_ext: bool) -> Vec { + let p = cs(&dir_form(prefix)); let mut count: i32 = 0; unsafe { if kvspaceListLen(kv, p, expand_ext as c_int, 1, &mut count) != 0 || count <= 0 { @@ -592,20 +592,35 @@ fn join_path(parent: &str, child: &str) -> String { format!("{}/{}", parent, child) } +// An extindex mount is an @ext value on the directory key (head.ptr=2), whose body +// holds the source path. Ignore langtype since backends differ on whether it is set. fn read_ext(kv: *mut c_void, prefix: &str) -> String { - let v = get(kv, prefix); + let v = get(kv, &dir_form(prefix)); if v.is_empty() { return String::new(); } let d = from_wire(&v); - if d.kind != "extindex" || d.body.is_empty() { + if d.r#ref != KVSPACE_XVALUE_PTR_EXT { return String::new(); } - // body = ext_path 前缀 + cap×M 成员矩阵;ext_path = body[..body_len − cap*M]。 - let cap = d.dims.get(1).copied().unwrap_or(0).max(0) as usize; - let m = d.dims.get(2).copied().unwrap_or(0).max(0) as usize; - let off = d.body.len().saturating_sub(cap * m); - String::from_utf8_lossy(&d.body[..off]).into_owned() + String::from_utf8_lossy(&d.body).into_owned() +} + +// Shared list/tree arguments: [prefix] [--kind[=false]] [--showext[=false]]; default prefix is `/`. +fn parse_walk_args(tail: &[String], show_kind_default: bool) -> (String, bool, bool) { + let mut show_ext = true; + let mut show_kind = show_kind_default; + let mut prefix = "/".to_string(); + for a in tail { + match a.as_str() { + "--kind" => show_kind = true, + "--kind=false" => show_kind = false, + "--showext" => show_ext = true, + "--showext=false" => show_ext = false, + _ => prefix = a.clone(), + } + } + (prefix, show_ext, show_kind) } fn print_help() { @@ -672,30 +687,27 @@ fn main() { if tail.len() != 2 { fatalf("usage: kvspace set "); } - let (val, _langtype) = parse_value(&tail[1]); - set_one(kv, &tail[0], &val); + set_one(kv, &tail[0], &parse_value(&tail[1])); } "head" => { + if tail.is_empty() { + fatalf("usage: kvspace head ..."); + } for k in tail { let v = get(kv, k); if v.is_empty() { println!("{}\t(nil)", k); } else { + // ndim/dims come from leading [dims] in langtype. let d = from_wire(&v); - let storetype = v[6]; - let ptr = v[7]; - let dims = &d.dims[..]; println!( "{}\t{}\tref={}\tstoretype={}\tndim={}\tdims=[{}]", k, d.kind, - ptr, - storetype, - dims.len(), - dims.iter() - .map(|x| x.to_string()) - .collect::>() - .join(",") + d.r#ref, + d.storetype, + d.dims.len(), + join_dims(&d.dims) ); } } @@ -770,11 +782,7 @@ fn main() { .first() .unwrap_or_else(|| fatalf("usage: kvspace mkindex [capacity]")); // 目录形归一:shm 自动补 /,durable 强校验 —— CLI 统一补,三后端一致。 - let p = if p.ends_with('/') || p.ends_with('·') { - p.to_string() - } else { - format!("{}/", p) - }; + let p = dir_form(p); let capacity: u32 = tail.get(1).and_then(|s| s.parse().ok()).unwrap_or(0); let mut err = [0u8; 256]; let rc = unsafe { @@ -809,56 +817,24 @@ fn main() { ) }; if rc != 0 { - let n = err.iter().position(|&b| b == 0).unwrap_or(err.len()); - if n > 0 { - fatalf(&String::from_utf8_lossy(&err[..n])); - } - // 部分后端不填 err(如 shm):给出可诊断的兜底信息。 - fatalf(&format!( - "extindex {} {}: failed rc={}(path/src_key 须目录形键且 src_key 须存在)", - tail[0], tail[1], rc - )); + // Fallback diagnostic when backend leaves err empty. + fail_op( + &err, + &format!( + "extindex {} {} (path/src_key must be directory keys and src_key must exist)", + tail[0], tail[1] + ), + ); } } "list" | "ls" => { - let mut show_ext = true; - let mut show_kind = true; - let mut prefix: Option<&str> = None; - for a in tail { - if a == "--kind" { - show_kind = true; - } else if a == "--kind=false" { - show_kind = false; - } else if a == "--showext" { - show_ext = true; - } else if a == "--showext=false" { - show_ext = false; - } else { - prefix = Some(a); - } - } - fprint_list(kv, prefix.unwrap_or("/"), show_ext, show_kind); + let (p, show_ext, show_kind) = parse_walk_args(tail, true); + fprint_list(kv, &p, show_ext, show_kind); } "tree" => { - let mut show_ext = true; - let mut show_kind = false; - let mut prefix: Option<&str> = None; - for a in tail { - if a == "--kind" { - show_kind = true; - } else if a == "--kind=false" { - show_kind = false; - } else if a == "--showext" { - show_ext = true; - } else if a == "--showext=false" { - show_ext = false; - } else { - prefix = Some(a); - } - } - let p = prefix.unwrap_or("/"); + let (p, show_ext, show_kind) = parse_walk_args(tail, false); println!("{}", p); - fprint_tree(kv, p, "", show_ext, show_kind); + fprint_tree(kv, &p, "", show_ext, show_kind); } "clear" => { let mut err = [0u8; 256]; @@ -878,31 +854,18 @@ fn main() { } } -fn strip_ext_children(kv: *mut c_void, prefix: &str, children: Vec) -> Vec { - let ext = read_ext(kv, prefix); - if ext.is_empty() { - return children; - } - let ext_children = list_names(kv, &ext, false); - let n = children.len().saturating_sub(ext_children.len()); - children[..n].to_vec() -} - fn fprint_list(kv: *mut c_void, prefix: &str, show_ext: bool, show_kind: bool) { - let mut children = list_names(kv, prefix, true); - if !show_ext { - children = strip_ext_children(kv, prefix, children); - } - for c in children { - let full = join_path(prefix, &c); - let v = get(kv, &full); + // When show_ext is false, list own children only and print the ext mount separately below. + for c in list_names(kv, prefix, show_ext) { + let v = get(kv, &join_path(prefix, &c)); if v.is_empty() { println!("{}", c); - } else if show_kind { - let d = from_wire(&v); + continue; + } + let d = from_wire(&v); + if show_kind { println!("{}\t{}\t{}", c, d.kind, plain(&d)); } else { - let d = from_wire(&v); println!("{}\t{}", c, plain(&d)); } } @@ -918,10 +881,8 @@ fn fprint_list(kv: *mut c_void, prefix: &str, show_ext: bool, show_kind: bool) { } fn fprint_tree(kv: *mut c_void, prefix: &str, indent: &str, show_ext: bool, show_kind: bool) { - let mut children = list_names(kv, prefix, true); - if !show_ext { - children = strip_ext_children(kv, prefix, children); - } + // When show_ext is false, list own children only and print the ext mount separately below. + let mut children = list_names(kv, prefix, show_ext); children.sort_by(|a, b| { let (aa, bb) = (a.trim_end_matches('/'), b.trim_end_matches('/')); if aa == bb { @@ -944,11 +905,12 @@ fn fprint_tree(kv: *mut c_void, prefix: &str, indent: &str, show_ext: bool, show let v = get(kv, &full); if v.is_empty() { println!("{}{}{}", indent, branch, c); - } else if show_kind { - let d = from_wire(&v); + continue; + } + let d = from_wire(&v); + if show_kind { println!("{}{}{}\t{}\t{}", indent, branch, c, d.kind, plain(&d)); } else { - let d = from_wire(&v); println!("{}{}{}\t{}", indent, branch, c, plain(&d)); } } diff --git a/include/kvspace/kvspace.h b/include/kvspace/kvspace.h index 430d9c7..19f9e8b 100644 --- a/include/kvspace/kvspace.h +++ b/include/kvspace/kvspace.h @@ -6,9 +6,11 @@ * shm://... → libkvspace-c.so.1 * 其余(redis/fs/s3)→ libkvspace_durable.so.1 * - * Wire: [len:u48 LE@0-5][storetype:u8@6][ptr:u8@7][body@8][langtype@8+len]. - * Head is fixed 8 bytes; body offset is always 8; langtype (plaintext UTF-8, - * incl. dims) is the value tail at 8+len. Single-key write — no attr key. + * Wire: [len:u48 LE@0-5][storetype:u8@6][ptr:u8@7][body@8][attr@8+len]. + * Head is fixed 8 bytes; body offset is always 8. attr = [attr_len:u16 LE] + * + records [tag:u8][len:u16 LE][data]; tag 1 = langtype (plaintext UTF-8, + * incl. leading [dims]). Empty attr = 2 bytes. Single-key write — no attr key. + * See kvspace/xvalue_head.h. */ #ifndef KVSPACE_H @@ -33,7 +35,7 @@ extern "C" { #define KVSPACE_STORETYPE_FIXED_LARGE 2 #define KVSPACE_STORETYPE_EXT 3 -/* XValue 元数据。langtype 取自 value 尾部 @8+len。 */ +/* XValue metadata. langtype is read from attr tag=1; ndim/dims are parsed from leading [dims]. */ typedef struct { uint16_t headlen; /* head 总字节数;body 起于偏移 headlen */ uint8_t ref; /* 存储位置:见 KVSPACE_REF_* */ @@ -57,13 +59,17 @@ void kvspaceClose(void *h); /* ── 单点读写 / 目录 ──────────────────────────────────────────── */ -/* 借用读:*out 指向后端常驻空间(shm mmap / durable 读缓存),生命周期到该 key 被写/删 - * 或 handle close,调用方不得 free。resolve=1 穿透 link。key 不存在/空值 → *out=NULL、 - * *out_len=0、返回 0。 - * 例外(volatile keys):`/vthread/*‥status` 每次读都重读后端,上一视图只保活到该 key - * 下一次读(见 spec CABI「volatile keys」)。 */ +/* Borrowed read: *out points into backend storage (shm mmap / durable read cache), + * valid until the key is overwritten/deleted or the handle is closed; do not free. + * Missing/empty key -> *out=NULL, *out_len=0, returns 0. + * resolve != 0 follows leaf Ptr chains (up to 16 hops; loops/dangling treated as missing); + * resolve == 0 returns the leaf wire as-is without following leaf Ptr. + * Exception (volatile keys): `/vthread//‥status` re-reads the backend on each call, + * keeping the previous view alive only until the next read of that key. */ int kvspaceGet(void *h, const char *key, int resolve, uint8_t **out, uint32_t *out_len); -/* Store a complete XValue before returning. */ +/* Full-value write: writes a complete XValue (validating wire header and attr bounds). + * Stores syntactically valid Ptr XValues at the wire layer without dereferencing the target + * or checking its current langtype (allowing multi-hop `slot -> ‥arg` bindings and forward/cyclic pointers). */ int kvspaceSetValue(void *h, const char *key, const uint8_t *value, uint32_t value_len, char *err, uint32_t err_cap); @@ -89,12 +95,11 @@ int kvspaceSetValueByRef(void *h, kvspaceRef_t *ref, const char *key_fallback, const uint8_t *value, uint32_t value_len, char *err, uint32_t err_cap); -/* shm-only slot view:把已绑定槽的映射内地址一次性交给调用方,供每步直读/直写。 - * `head` 指向 XValue 起点,`body = head + headlen`;`cap` 是整 box 物理容量 - * (allocator 实际值,不是请求值),`body_cap = cap - headlen`。借用模型: - * 寿命到该 key 被写/删、view 失效或 handle close。 - * 返回 0=成功;-1=不存在/ref 失效/不可视槽;-2=后端不支持(调用方回退既有 API, - * 每句柄只判定一次)。 */ +/* shm-only slot view: exposes mapped slot addresses for direct per-step read/write. + * `head` points to XValue start, `body = head + headlen`; `cap` is physical box capacity, + * `body_cap = cap - headlen`. Borrowed until key overwrite/delete, view invalidation, or close. + * Returns 0 on success, -1 if missing/invalid/non-viewable, -2 if unsupported by backend + * (including when the backend does not export this symbol; caller falls back once per handle). */ typedef struct { uint8_t *head; uint32_t headlen; @@ -149,15 +154,16 @@ int kvspaceSetPart(void *h, const char *key, uint32_t offset, const uint8_t *buf /* Read the wire head + langtype tail. Missing keys return nonzero. */ int kvspaceGetHead(void *h, const char *key, kvspaceHead_t *out); -/* 就地写:key 必须已存在、kind 不变、body_len 必须等于原 body_len——返回原 box 的 body - * 偏移指针供调用方直接写。违反前置条件 → 非 0 + err(绝不静默重分配、绝不回落)。 - * 落盘时机:body 由调用方在返回后填,故本笔在下一次 kvspace* 调用(读也算)或 kvspaceClose - * 时落盘;前者失败以返回码 + err 报出,后者失败写 stderr。 */ +/* In-place write: key must exist with unchanged kind and body_len; returns existing body + * pointer for direct mutation, or nonzero + err on precondition failure (never reallocates). + * resolve != 0 follows leaf Ptr chains (up to 16 hops) and writes the terminal target in place; + * resolve == 0 operates on the leaf itself without following leaf Ptr. + * Flushed on the next kvspace* call or kvspaceClose. */ int kvspaceWriteInPlace(void *h, const char *key, int resolve, uint32_t body_len, uint8_t **body, char *err, uint32_t err_cap); /* Reserve a new XValue and return its body pointer; fill it before another KVSpace call. - * langtype(可为空 = 无类型)随头/尾部整体写入——单 key 一次写。 */ + * langtype (empty/NULL = untyped) is written with the header/attr in a single-key write. */ int kvspaceWriteNewPlace(void *h, const char *key, uint8_t ref, uint8_t storetype, uint32_t body_len, uint64_t body_cap, const char *langtype, uint8_t **body, char *err, uint32_t err_cap); @@ -185,10 +191,13 @@ int kvspaceClear(void *h, char *err, uint32_t err_cap); int kvspaceWatch(void *h, const char *key, const uint8_t *target, uint32_t target_len, uint64_t tick_ns, uint8_t **out, uint32_t *out_len); -/* ── XValue 构造(无 handle,由前端静态实现,byte-identical)─────────── - * head 固定 8 字节,直接 memcpy struct + body + langtype 尾部 - * (见 kvspace/xvalue_head.h)。langtype 为空串 = 无类型(不追加尾部)。 */ +/* XValue constructors (stateless, implemented in frontend, byte-identical across backends). + * Fixed 8-byte header + body + attr region (see kvspace/xvalue_head.h). + * Empty/NULL langtype = untyped (omits langtype record; attr region is just 2-byte attr_len). */ +/* Construct a leaf Ptr XValue (storetype=SLACK, ptr=PTR, body=target path bytes, + * target_kindexpr written to attr tag=1; "" means unconstrained; rejects "any" and empty target). + * Pure wire constructor: does not connect to a backend or validate target existence/langtype. */ int kvspaceNewPtr(const char *target_kindexpr, const char *target, uint8_t **out, uint32_t *out_len); int kvspaceNewChar(const uint8_t *bytes, uint32_t len, const char *langtype, diff --git a/include/kvspace/xvalue_head.h b/include/kvspace/xvalue_head.h index c620d6f..3b12a12 100644 --- a/include/kvspace/xvalue_head.h +++ b/include/kvspace/xvalue_head.h @@ -26,8 +26,8 @@ extern "C" { #define KVSPACE_XVALUE_PTR_PTR 1u #define KVSPACE_XVALUE_PTR_EXT 2u -/* 固定 8 字节布局(直接 memcpy struct + body)。 - wire:len:u48@0-5(小端)、storetype@6、ptr@7、body@8。 */ +/* Fixed 8-byte layout (direct memcpy struct + body). + wire: len:u48@0-5 (little-endian), storetype@6, ptr@7, body@8. */ typedef struct kvspaceXvalueHead { uint8_t len[6]; /* u48 小端(offset 0-5) */ uint8_t storetype; @@ -51,14 +51,18 @@ static inline void kvspaceXvalueHeadSetLen(kvspaceXvalueHead *h, uint64_t v) { /* attr 区常量:record tag 1 = langtype;2+ 预留扩展(ro/vid 等,未实现)。 */ #define KVSPACE_XVALUE_ATTR_TAG_LANGTYPE 1u -/* 读 attr 总长(含自身 2 字节;空 attr = 2)。 */ +/* Read total attr length (including its own 2-byte length prefix; empty attr = 2). + Unbounded: caller must ensure wire has at least 8+body_len+2 bytes; use + kvspaceXvalueHeadLangtypeBounded for untrusted or backend-borrowed inputs. */ static inline uint16_t kvspaceXvalueHeadAttrLen(const uint8_t *wire, uint64_t body_len) { const uint8_t *p = wire + KVSPACE_XVALUE_HEAD_LEN + body_len; return (uint16_t)((uint16_t)p[0] | ((uint16_t)p[1] << 8)); } -/* 扫 attr records 取 tag=1(langtype)记录 data;未找到/坏 attr → 返回 0。 */ +/* Scan attr records for tag=1 (langtype) data; returns 0 if missing or malformed. + Unbounded: caller must ensure wire has at least 8+body_len+attr_len bytes; use + kvspaceXvalueHeadLangtypeBounded when bounds are not yet verified. */ static inline uint16_t kvspaceXvalueHeadLangtype(const uint8_t *wire, uint64_t body_len, const uint8_t **out) { @@ -82,8 +86,31 @@ static inline uint16_t kvspaceXvalueHeadLangtype(const uint8_t *wire, return 0; } -/* 构造(固定头 + body + attr 区,单 key 整体)。langtype 空/NULL = 无 langtype - 记录(空 attr = [attr_len:2])。0 on success. -1 and *out cleared on failure. */ +/* Bounded variant: verifies 8+len+2 <= wire_len and 8+len+attr_len <= wire_len + before scanning records. Returns 0 and sets *out=NULL on missing/malformed/truncated values. */ +static inline uint16_t kvspaceXvalueHeadLangtypeBounded(const uint8_t *wire, + uint64_t wire_len, + const uint8_t **out) { + *out = NULL; + if (!wire || wire_len < KVSPACE_XVALUE_HEAD_LEN + 2u) + return 0; + uint64_t body_len = 0; + for (int i = 5; i >= 0; i--) + body_len = (body_len << 8) | wire[i]; + const uint64_t room = wire_len - KVSPACE_XVALUE_HEAD_LEN; + if (body_len > room - 2u) + return 0; + if (kvspaceXvalueHeadAttrLen(wire, body_len) > room - body_len) + return 0; + return kvspaceXvalueHeadLangtype(wire, body_len, out); +} + +/* Constructors (fixed 8-byte header + body + attr region in a single value). + Empty/NULL langtype omits the langtype record (empty attr = [attr_len:2]). + Returns 0 on success, -1 and clears *out on failure (when out is non-NULL). + Fixed header has no capacity/shape fields: cap only checks cap >= len, and + dims/ndim only check non-empty (shape lives in leading [dims] of langtype). + NewCode is an alias of NewSlack (SLACK/INLINE) kept for ABI compatibility. */ int kvspaceXvalueHeadNewNone(uint8_t **out, uint64_t *out_len); int kvspaceXvalueHeadNewScalar(uint8_t storetype, const uint8_t *raw, uint32_t raw_len, const char *langtype, diff --git a/kvspace.pc.in b/kvspace.pc.in index b5fcff0..317352d 100644 --- a/kvspace.pc.in +++ b/kvspace.pc.in @@ -1,6 +1,6 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@/kvspace includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: kvspace diff --git a/src/frontend.c b/src/frontend.c index 10ab8e6..8dcc14f 100644 --- a/src/frontend.c +++ b/src/frontend.c @@ -96,7 +96,6 @@ static kvspace_handle *H(void *h) { return (kvspace_handle *)h; } /* 活跃句柄表:durable 的写侧惰性(body 由调用方在返回后填,落盘延到下一次 op),故漏调 * Close 就退出会丢最后一笔写。进程正常退出时前端兜底落盘再关闭,兑现 kvspace.h 的退出保证。 */ static kvspace_handle *live; -static int live_hooked; static pthread_mutex_t live_lock = PTHREAD_MUTEX_INITIALIZER; static void handle_free(kvspace_handle *x) { @@ -114,9 +113,14 @@ static void live_unlink(kvspace_handle *x) { pthread_mutex_unlock(&live_lock); } -/* 退出兜底:未 Close 句柄逐个关闭——durable 的 Close 落盘未决写,落盘失败由后端自己写 - * stderr。数据可以丢(后端故障),但绝不静默。 */ -static void exit_close(void) { +/* Exit fallback: close any unclosed handles so durable backends flush pending writes + * (reporting errors to stderr). + * + * Uses an ELF destructor rather than atexit: atexit handlers run in LIFO order, so if + * the host registered its own cleanup before the first Connect, an atexit handler here + * would free handles before the host cleanup runs (causing use-after-free). Destructors + * run after all atexit handlers (_dl_fini / dyld finalizer), and dlclose is safe there. */ +__attribute__((destructor)) static void exit_close(void) { for (;;) { pthread_mutex_lock(&live_lock); kvspace_handle *x = live; @@ -237,8 +241,9 @@ void *kvspaceConnect(const char *dsn) { LOAD(rmindexext, "kvspaceRmindexExt"); LOAD(clear, "kvspaceClear"); LOAD(watch, "kvspaceWatch"); - LOAD(xvalue_body_view, "kvspaceXvalueBodyView"); #undef LOAD + /* Optional symbol: trampoline returns -2 (unsupported) when absent. */ + *(void **)&vt->xvalue_body_view = dlsym(dl, "kvspaceXvalueBodyView"); *(void **)&vt->resolveref = dlsym(dl, "kvspaceResolveRef"); *(void **)&vt->getbyref = dlsym(dl, "kvspaceGetByRef"); *(void **)&vt->setpartbyref = dlsym(dl, "kvspaceSetPartByRef"); @@ -271,7 +276,6 @@ void *kvspaceConnect(const char *dsn) { pthread_mutex_lock(&live_lock); h->next = live; live = h; - if (!live_hooked) { live_hooked = 1; atexit(exit_close); } pthread_mutex_unlock(&live_lock); return h; } @@ -326,6 +330,7 @@ int kvspaceSetPartByRef(void *h, kvspaceRef_t *ref, const char *key_fallback, int kvspaceXvalueBodyView(void *h, const kvspaceRef_t *ref, const char *key_fallback, kvspaceXvalueBodyView_t *out) { kvspace_handle *x = H(h); + if (!x->vt->xvalue_body_view) return -2; return x->vt->xvalue_body_view(x->backend, ref, key_fallback, out); } diff --git a/src/xvalue_head.c b/src/xvalue_head.c index f11f6ba..d7dd7cc 100644 --- a/src/xvalue_head.c +++ b/src/xvalue_head.c @@ -3,17 +3,26 @@ #include #include -/* V2 最终态(#571):head 固定 8 字节 [len:u48 LE@0-5][storetype:u8@6][ptr:u8@7], - body 偏移恒 = 8;attr 区 @8+len(长度自描述,含 langtype 记录)。整体单 key - 一次写,天然原子——无 attr key。langtype 空 = 无记录(空 attr)。 */ +/* V2 final layout (#571): fixed 8-byte header [len:u48 LE@0-5][storetype:u8@6][ptr:u8@7], + body always at offset 8, self-delimited attr region at 8+len (carrying langtype). + Single-key atomic write — no separate attr key. Empty langtype = empty attr. */ #define UINT48_MAX UINT64_C(0xFFFFFFFFFFFF) +/* Failure helper: clear *out per header contract. */ +static int fail(uint8_t **out) { + if (out) + *out = NULL; + return -1; +} + +/* Single wire builder. Zero-fills body when body is NULL (Reserve placeholder); + public entry points validate whether non-zero len requires non-NULL body. */ static int build(uint8_t storetype, uint8_t ptr, const uint8_t *body, uint64_t len, const char *langtype, uint8_t **out, uint64_t *out_len) { - if (!out || !out_len || (len && !body)) - return -1; + if (!out || !out_len) + return fail(out); *out = NULL; if (storetype > KVSPACE_XVALUE_ST_EXT || ptr > KVSPACE_XVALUE_PTR_EXT) return -1; @@ -32,7 +41,7 @@ static int build(uint8_t storetype, uint8_t ptr, const uint8_t *body, kvspaceXvalueHead h = {.storetype = storetype, .ptr = ptr}; kvspaceXvalueHeadSetLen(&h, len); memcpy(p, &h, sizeof h); - if (len) + if (len && body) memcpy(p + KVSPACE_XVALUE_HEAD_LEN, body, len); uint8_t *a = p + KVSPACE_XVALUE_HEAD_LEN + len; a[0] = (uint8_t)alen; @@ -57,7 +66,7 @@ int kvspaceXvalueHeadNewScalar(uint8_t storetype, const uint8_t *raw, uint32_t raw_len, const char *langtype, uint8_t **out, uint64_t *out_len) { if (raw_len && !raw) - return -1; + return fail(out); return build(storetype, KVSPACE_XVALUE_PTR_INLINE, raw, raw_len, langtype, out, out_len); } @@ -65,9 +74,8 @@ int kvspaceXvalueHeadNewScalar(uint8_t storetype, const uint8_t *raw, int kvspaceXvalueHeadNewSlack(const uint8_t *data, uint64_t data_len, uint64_t cap, const char *langtype, uint8_t **out, uint64_t *out_len) { - if (cap < data_len || data_len > UINT48_MAX || (data_len && !data)) - return -1; - (void)cap; + if (cap < data_len || (data_len && !data)) + return fail(out); return build(KVSPACE_XVALUE_ST_SLACK, KVSPACE_XVALUE_PTR_INLINE, data, data_len, langtype, out, out_len); } @@ -76,12 +84,9 @@ int kvspaceXvalueHeadNewTensor(const uint64_t *dims, uint32_t ndim, const uint8_t *raw, uint64_t raw_len, const char *langtype, uint8_t **out, uint64_t *out_len) { - if (!dims || ndim == 0 || ndim > 64 || raw_len > UINT48_MAX || - (raw_len && !raw)) - return -1; - (void)dims; - (void)ndim; - /* 形状在 langtype 尾部;body 字节数 = raw_len(明文层校验 numel×width) */ + /* Shape lives in leading [dims] of langtype; body byte length is raw_len. */ + if (!dims || ndim == 0 || ndim > 64 || (raw_len && !raw)) + return fail(out); return build(KVSPACE_XVALUE_ST_LARGE, KVSPACE_XVALUE_PTR_INLINE, raw, raw_len, langtype, out, out_len); } @@ -90,7 +95,7 @@ int kvspaceXvalueHeadNewPtr(const uint8_t *path, uint32_t path_len, const char *langtype, uint8_t **out, uint64_t *out_len) { if (!path_len || !path) - return -1; + return fail(out); return build(KVSPACE_XVALUE_ST_SLACK, KVSPACE_XVALUE_PTR_PTR, path, path_len, langtype, out, out_len); } @@ -99,7 +104,7 @@ int kvspaceXvalueHeadNewExt(const uint8_t *locator, uint32_t locator_len, const char *langtype, uint8_t **out, uint64_t *out_len) { if (!locator_len || !locator) - return -1; + return fail(out); return build(KVSPACE_XVALUE_ST_EXT, KVSPACE_XVALUE_PTR_EXT, locator, locator_len, langtype, out, out_len); } @@ -107,46 +112,17 @@ int kvspaceXvalueHeadNewExt(const uint8_t *locator, uint32_t locator_len, int kvspaceXvalueHeadNewCode(const uint8_t *body, uint64_t body_len, uint64_t cap, const char *langtype, uint8_t **out, uint64_t *out_len) { - if (body_len > cap || body_len > UINT48_MAX || (body_len && !body)) - return -1; - (void)cap; - return build(KVSPACE_XVALUE_ST_SLACK, KVSPACE_XVALUE_PTR_INLINE, body, - body_len, langtype, out, out_len); + return kvspaceXvalueHeadNewSlack(body, body_len, cap, langtype, out, + out_len); } int kvspaceXvalueHeadReserve(uint8_t storetype, uint64_t body_len, uint64_t cap, const char *langtype, uint8_t **out, uint64_t *out_len) { - if (body_len > UINT48_MAX || cap < body_len) - return -1; /* 固定头无容量字段:预留 = 产 body_len 字节零 body 的占位(后由调用方填充), attr 区(含 langtype 记录)一并写入。 */ - (void)cap; - uint64_t ltl = langtype ? strlen(langtype) : 0; - uint64_t alen = 2 + (ltl ? 3 + ltl : 0); - if (alen > UINT16_MAX) - return -1; - if (body_len > SIZE_MAX - KVSPACE_XVALUE_HEAD_LEN - alen) - return -1; - uint64_t n = (uint64_t)KVSPACE_XVALUE_HEAD_LEN + body_len + alen; - if (n > SIZE_MAX) - return -1; - uint8_t *p = (uint8_t *)calloc(1, (size_t)n); - if (!p) - return -1; - kvspaceXvalueHead h = {.storetype = storetype, .ptr = KVSPACE_XVALUE_PTR_INLINE}; - kvspaceXvalueHeadSetLen(&h, body_len); - memcpy(p, &h, sizeof h); - uint8_t *a = p + KVSPACE_XVALUE_HEAD_LEN + body_len; - a[0] = (uint8_t)alen; - a[1] = (uint8_t)(alen >> 8); - if (ltl) { - a[2] = KVSPACE_XVALUE_ATTR_TAG_LANGTYPE; - a[3] = (uint8_t)ltl; - a[4] = (uint8_t)(ltl >> 8); - memcpy(a + 5, langtype, ltl); - } - *out = p; - *out_len = n; - return 0; + if (cap < body_len) + return fail(out); + return build(storetype, KVSPACE_XVALUE_PTR_INLINE, NULL, body_len, + langtype, out, out_len); } diff --git a/tests/test_xvalue_head.c b/tests/test_xvalue_head.c index 26f0b6b..31a1656 100644 --- a/tests/test_xvalue_head.c +++ b/tests/test_xvalue_head.c @@ -168,6 +168,167 @@ static void test_reserve_code(void) { "code cap KVSPACE_XVALUE_ST_EXT); + - LangtypeBounded returns the exact non-NUL-terminated slice for valid values + and 0 with *out = NULL on truncated/malformed inputs without reading out of bounds. */ +static void test_bounded_and_reserve_fail(void) { + uint8_t sentinel = 0x5a; + uint8_t *buf = &sentinel; + uint64_t n = 0; + const uint8_t *lt = &sentinel; + + /* Reserve with invalid storetype (4, 9, 255) or cap < body_len: reject and clear *out */ + CHECK(kvspaceXvalueHeadReserve(4, 4, 4, "int32", &buf, &n) == -1 && buf == NULL, + "Reserve(storetype=4) rejected and clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadReserve(9, 4, 4, "int32", &buf, &n) == -1 && buf == NULL, + "Reserve(storetype=9) rejected and clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadReserve(255, 0, 0, "", &buf, &n) == -1 && buf == NULL, + "Reserve(storetype=255) rejected and clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadReserve(KVSPACE_XVALUE_ST_SLACK, 8, 4, "", &buf, &n) == -1 && + buf == NULL, + "Reserve(cap < body_len) rejected and clears *out"); + CHECK(kvspaceXvalueHeadReserve(KVSPACE_XVALUE_ST_SLACK, 4, 4, "", NULL, &n) == -1, + "Reserve(out=NULL) rejected"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadReserve(KVSPACE_XVALUE_ST_SLACK, 4, 4, "", &buf, NULL) == -1 && + buf == NULL, + "Reserve(out_len=NULL) rejected and clears *out"); + + /* Other constructors also clear *out on failure */ + buf = &sentinel; + CHECK(kvspaceXvalueHeadNewScalar(KVSPACE_XVALUE_ST_SHORT, NULL, 3, "", &buf, &n) == -1 && + buf == NULL, + "NewScalar(NULL, 3) clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadNewScalar(4, &sentinel, 1, "", &buf, &n) == -1 && buf == NULL, + "NewScalar(storetype=4) clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadNewSlack(NULL, 4, 4, "", &buf, &n) == -1 && buf == NULL, + "NewSlack(NULL, 4) clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadNewTensor(NULL, 0, &sentinel, 1, "", &buf, &n) == -1 && buf == NULL, + "NewTensor(NULL dims) clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadNewPtr(NULL, 0, "", &buf, &n) == -1 && buf == NULL, + "NewPtr(empty) clears *out"); + buf = &sentinel; + CHECK(kvspaceXvalueHeadNewExt(NULL, 0, "", &buf, &n) == -1 && buf == NULL, + "NewExt(empty) clears *out"); + + /* LangtypeBounded valid inputs: scalar / tensor / ptr / empty langtype */ + uint8_t raw8[8] = {1, 0, 0, 0, 0, 0, 0, 0}; + CHECK(kvspaceXvalueHeadNewScalar(KVSPACE_XVALUE_ST_SHORT, raw8, 8, "int64", &buf, &n) == 0, + "scalar build"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, n, <) == 5 && lt != NULL && + memcmp(lt, "int64", 5) == 0, + "bounded scalar langtype"); + free(buf); + + uint64_t dims[2] = {2, 3}; + uint8_t raw24[24] = {0}; + CHECK(kvspaceXvalueHeadNewTensor(dims, 2, raw24, sizeof raw24, "[2,3]float32", &buf, &n) == 0, + "tensor build"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, n, <) == 12 && lt != NULL && + memcmp(lt, "[2,3]float32", 12) == 0, + "bounded tensor langtype"); + free(buf); + + CHECK(kvspaceXvalueHeadNewPtr((const uint8_t *)"/dst", 4, "[]char/utf8", &buf, &n) == 0, + "ptr build"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, n, <) == 11 && lt != NULL && + memcmp(lt, "[]char/utf8", 11) == 0, + "bounded ptr langtype"); + free(buf); + + CHECK(kvspaceXvalueHeadNewNone(&buf, &n) == 0, "none build"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, n, <) == 0 && lt == NULL, + "bounded empty langtype returns 0 and *out=NULL"); + free(buf); + + /* LangtypeBounded malformed / truncated boundary cases */ + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(NULL, 16, <) == 0 && lt == NULL, + "bounded NULL wire"); + + uint8_t short_wire[9] = {0}; + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(short_wire, 0, <) == 0 && lt == NULL, + "bounded wire_len=0"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(short_wire, 9, <) == 0 && lt == NULL, + "bounded wire_len=9 (< 10)"); + + /* body_len > wire_len - 10 (legacy wire without attr tail or truncated body) */ + CHECK(kvspaceXvalueHeadReserve(KVSPACE_XVALUE_ST_SLACK, 4, 4, "int32", &buf, &n) == 0, + "reserve for truncation checks"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, KVSPACE_XVALUE_HEAD_LEN + 4, <) == 0 && + lt == NULL, + "bounded rejects wire without 2-byte attr_len (body_len > wire_len - 10)"); + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, KVSPACE_XVALUE_HEAD_LEN + 4 + 1, <) == 0 && + lt == NULL, + "bounded rejects 1-byte truncated attr_len"); + /* attr_len exceeds remaining room (truncate last byte) */ + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(buf, n - 1, <) == 0 && lt == NULL, + "bounded rejects attr_len exceeding remaining room"); + free(buf); + + /* attr_len < 2 */ + uint8_t bad_alen[10] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 0}; + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(bad_alen, sizeof bad_alen, <) == 0 && lt == NULL, + "bounded rejects attr_len < 2"); + + /* Truncated record: attr_len fits in wire_len, but p + 3 + rlen > end */ + uint8_t trunc_rec[15] = { + 0, 0, 0, 0, 0, 0, 0, 0, /* head: body_len=0 */ + 7, 0, /* attr_len = 7 (end = 15) */ + KVSPACE_XVALUE_ATTR_TAG_LANGTYPE, /* tag = 1 */ + 10, 0, /* rlen = 10 -> p+3+10 = 23 > 15 */ + 'i', '6' + }; + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(trunc_rec, sizeof trunc_rec, <) == 0 && lt == NULL, + "bounded rejects truncated attr record (p + 3 + rlen > end)"); + + /* Trailing record header shorter than 3 bytes (alen = 4 leaves 2 bytes) */ + uint8_t short_rec_hdr[12] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, + KVSPACE_XVALUE_ATTR_TAG_LANGTYPE, 0 + }; + lt = &sentinel; + CHECK(kvspaceXvalueHeadLangtypeBounded(short_rec_hdr, sizeof short_rec_hdr, <) == 0 && + lt == NULL, + "bounded rejects <3B trailing record header"); + + /* Non-NUL-terminated bounded slice after skipping an unknown tag: + record 1 is tag=2 (reserved), record 2 is tag=1 (langtype="int64") immediately + followed by non-NUL byte 9 (next record tag); verify exact rlen=5 slice. */ + uint8_t multi_non_nul[8 + 0 + 2 + (3 + 2) + (3 + 5) + (3 + 1)] = { + 0, 0, 0, 0, 0, 0, 0, 0, /* head: body_len=0 */ + 19, 0, /* attr_len = 2 + 5 + 8 + 4 = 19 */ + 2, 2, 0, 0xaa, 0xbb, /* tag=2, rlen=2 */ + KVSPACE_XVALUE_ATTR_TAG_LANGTYPE, 5, 0, 'i', 'n', 't', '6', '4', /* tag=1, rlen=5 (followed by non-zero) */ + 9, 1, 0, 'Z' /* tag=9, rlen=1 -> lt[5] == 9 != '\0' */ + }; + lt = &sentinel; + uint16_t got_len = kvspaceXvalueHeadLangtypeBounded(multi_non_nul, sizeof multi_non_nul, <); + CHECK(got_len == 5 && lt != NULL && lt[5] == 9 && memcmp(lt, "int64", 5) == 0, + "bounded returns exact non-NUL-terminated slice after skipping unknown tag"); +} + int main(void) { test_none(); test_scalar(); @@ -175,6 +336,7 @@ int main(void) { test_tensor(); test_ptr_ext(); test_reserve_code(); + test_bounded_and_reserve_fail(); if (failures) { fprintf(stderr, "%d failed\n", failures); return 1; diff --git a/tutorial-c/01_basic.c b/tutorial-c/01_basic.c index 6be24e4..f7e8a32 100644 --- a/tutorial-c/01_basic.c +++ b/tutorial-c/01_basic.c @@ -5,8 +5,8 @@ * # === Set & Get === * # /t01/a int64:42 * # === Set & List === - * # a int64 - * # b int64 + * # a int64 42 + * # b int64 7 * # c char/utf8 * # === Get bulk === * # /t01/a int64:42 @@ -16,8 +16,8 @@ * # /t01/nonexist (nil) * # === Del === * # /t01/a (nil) - * # b int64 - * # c char/utf8 + * # b + * # c * # /end */ #include "common.h" @@ -27,7 +27,7 @@ static void get(void *kv, const char *key) { uint32_t len; uint8_t *v = kv_get(kv, key, &len); if (!v || len == 0) { printf("%s\t(nil)\n", key); return; } char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, key, v, len, kind, sizeof(kind), &body, &bl); + read_kind(v, len, kind, sizeof(kind), &body, &bl); printf("%s\t%s:", key, kind); if (strcmp(kind, "int64") == 0 && bl >= 8) printf("%ld", *(const int64_t *)body); else if (strcmp(kind, "char/utf8") == 0) printf("%.*s", (int)bl, body); @@ -58,7 +58,7 @@ static void list(void *kv, const char *dir, int show_kind) { char full[512]; snprintf(full, sizeof(full), "%s%s", dir, name); uint32_t vl; uint8_t *v = kv_get(kv, full, &vl); if (v) { char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, full, v, vl, kind, sizeof(kind), &body, &bl); + read_kind(v, vl, kind, sizeof(kind), &body, &bl); printf("%s\t%s", name, kind); if (strcmp(kind, "int64") == 0 && bl >= 8) printf("\t%ld", *(const int64_t *)body); printf("\n"); diff --git a/tutorial-c/02_cpp.cpp b/tutorial-c/02_cpp.cpp index b4835e3..31dd3db 100644 --- a/tutorial-c/02_cpp.cpp +++ b/tutorial-c/02_cpp.cpp @@ -1,7 +1,7 @@ /* * 02_cpp — C++ RAII 封装:extern "C" 直接调用 libkvspace dispatch 前端 * - * #566:head 固定 8 字节(body 偏移恒 8),类型走 x‥langtype attr key。 + * #571: Fixed 8-byte header (body offset is always 8); langtype in inline attr region. * * 编译: g++ -std=c++17 02_cpp.cpp -lkvspace */ @@ -31,8 +31,7 @@ struct KV { } ~KV() { kvspaceClose(kv); ::unlink(path.c_str()); } - // 写即构造:memcpy 固定头取 body_len/storetype/ptr,就地/新位置二选一原语, - // 往返回指针写 body。langtype 由调用方显式给(写 x‥langtype attr key)。 + // Write into returned body pointer; WriteNewPlace writes langtype with the header/attr region. void set(const char *key, const char *langtype, const uint8_t *val, uint32_t len) { if (!val || len < KVSPACE_XVALUE_HEAD_LEN) return; kvspaceXvalueHead hd; @@ -40,24 +39,24 @@ struct KV { uint64_t bl = kvspaceXvalueHeadLen(&hd); uint8_t *dst = nullptr; char err[256]; if (kvspaceWriteInPlace(kv, key, 1, (uint32_t)bl, &dst, err, sizeof(err)) != 0) - kvspaceWriteNewPlace(kv, key, hd.ptr, hd.storetype, langtype, - (uint32_t)bl, bl, &dst, err, sizeof(err)); + kvspaceWriteNewPlace(kv, key, hd.ptr, hd.storetype, (uint32_t)bl, bl, + langtype, &dst, err, sizeof(err)); if (bl && dst) memcpy(dst, val + KVSPACE_XVALUE_HEAD_LEN, (size_t)bl); } - // 借用读:out 指向 kvspace 常驻空间,拷进 std::string 后即失效,不 free。 - // kind 由 kvspaceGetHead 从 x‥langtype attr key 读。 + // Borrowed read: kind is decoded by kvspaceGetHead from inline attr; body length is head.body_len. bool get(const char *key, std::string &kind, std::string &raw) { uint8_t *out = nullptr; uint32_t len = 0; kvspaceGet(kv, key, 0, &out, &len); - if (!out || len == 0) return false; + if (!out || len < KVSPACE_XVALUE_HEAD_LEN) return false; kvspaceHead_t h; if (kvspaceGetHead(kv, key, &h) != 0) return false; const char *k = (const char *)h.langtype; if (k[0] == '*' || k[0] == '@') k++; if (k[0] == '[') { const char *e = strchr(k, ']'); if (e) k = e + 1; } kind = k; - raw.assign((const char *)(out + KVSPACE_XVALUE_HEAD_LEN), - len > KVSPACE_XVALUE_HEAD_LEN ? len - KVSPACE_XVALUE_HEAD_LEN : 0); + uint32_t bl = h.body_len > 0 ? (uint32_t)h.body_len : 0; + if (bl > len - KVSPACE_XVALUE_HEAD_LEN) return false; + raw.assign((const char *)(out + KVSPACE_XVALUE_HEAD_LEN), bl); return true; } void del(const char *key) { @@ -82,11 +81,12 @@ struct KV { int main() { auto xv_int = [](int64_t v) { - uint8_t *b; uint32_t n; kvspaceNewInt64(v, &b, &n); + uint8_t *b; uint32_t n; kvspaceNewInt64(v, "int64", &b, &n); return std::pair{b, n}; }; auto xv_str = [](const char *s) { - uint8_t *b; uint32_t n; kvspaceNewChar((const uint8_t *)s, (uint32_t)strlen(s), &b, &n); + uint8_t *b; uint32_t n; + kvspaceNewChar((const uint8_t *)s, (uint32_t)strlen(s), "char/utf8", &b, &n); return std::pair{b, n}; }; diff --git a/tutorial-c/03_python.py b/tutorial-c/03_python.py index cf56c2b..e3cf788 100644 --- a/tutorial-c/03_python.py +++ b/tutorial-c/03_python.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """03_python — ctypes 直调 libkvspace(dispatch 前端),shm:// 后端 -#566:head 固定 8 字节(body 偏移恒 8),类型走 x‥langtype attr key。""" +#571: Fixed 8-byte header (body offset is always 8); langtype in inline attr region.""" import ctypes import os @@ -17,7 +17,7 @@ KVSPACE_XVALUE_HEAD_LEN = 8 -# 与 kvspaceHead_t 对齐(langtype 由 kvspaceGetHead 从 attr key 填)。 +# Matches kvspaceHead_t (langtype populated by kvspaceGetHead from inline attr). class Head(ctypes.Structure): _fields_ = [ ("headlen", ctypes.c_uint16), @@ -61,8 +61,9 @@ def bind(fn, args, restype): bind(lib.kvspaceGetHead, [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(Head)], ctypes.c_int) # 写即构造:就地/新位置二选一原语,往返回的 body 偏移指针写字节。 bind(lib.kvspaceWriteInPlace, [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_uint32, ctypes.POINTER(U8P), ctypes.c_char_p, ctypes.c_uint32], ctypes.c_int) +# Signature matches kvspace.h: (h, key, ref, storetype, body_len, body_cap, langtype, body, err, err_cap). bind(lib.kvspaceWriteNewPlace, [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_uint8, ctypes.c_uint8, - ctypes.c_char_p, ctypes.c_uint32, ctypes.c_uint64, + ctypes.c_uint32, ctypes.c_uint64, ctypes.c_char_p, ctypes.POINTER(U8P), ctypes.c_char_p, ctypes.c_uint32], ctypes.c_int) # 前缀枚举:ListLen 定计数 + 逐 idx ListAt 借用取名。 bind(lib.kvspaceListLen, [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int32)], ctypes.c_int) @@ -71,25 +72,33 @@ def bind(fn, args, restype): bind(lib.kvspaceDel, [ctypes.c_void_p, CCHARPP, ctypes.c_uint32, ctypes.c_char_p, ctypes.c_uint32], ctypes.c_int) bind(lib.kvspaceDelTree, [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint32], ctypes.c_int) bind(lib.kvspaceMkindex, [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_uint32, ctypes.c_char_p, ctypes.c_uint32], ctypes.c_int) -bind(lib.kvspaceNewInt64, [ctypes.c_int64, ctypes.POINTER(U8P), U32P], ctypes.c_int) -bind(lib.kvspaceNewChar, [U8P, ctypes.c_uint32, ctypes.POINTER(U8P), U32P], ctypes.c_int) +# New*: langtype is stored in inline attr (empty/None = untyped). +bind(lib.kvspaceNewInt64, [ctypes.c_int64, ctypes.c_char_p, ctypes.POINTER(U8P), U32P], ctypes.c_int) +bind(lib.kvspaceNewChar, [U8P, ctypes.c_uint32, ctypes.c_char_p, ctypes.POINTER(U8P), U32P], ctypes.c_int) +bind(lib.free, [ctypes.c_void_p], None) def enc_int(v): out, n = U8P(), ctypes.c_uint32() - lib.kvspaceNewInt64(v, ctypes.byref(out), ctypes.byref(n)) - return ctypes.string_at(out, n.value) + if lib.kvspaceNewInt64(v, b"int64", ctypes.byref(out), ctypes.byref(n)) != 0: + raise RuntimeError("kvspaceNewInt64 failed") + data = ctypes.string_at(out, n.value) + lib.free(out) # allocated by frontend malloc + return data def new_str(s): b = s.encode() buf = (ctypes.c_uint8 * len(b)).from_buffer_copy(b) out, n = U8P(), ctypes.c_uint32() - lib.kvspaceNewChar(buf, len(b), ctypes.byref(out), ctypes.byref(n)) - return ctypes.string_at(out, n.value) + if lib.kvspaceNewChar(buf, len(b), b"char/utf8", ctypes.byref(out), ctypes.byref(n)) != 0: + raise RuntimeError("kvspaceNewChar failed") + data = ctypes.string_at(out, n.value) + lib.free(out) # allocated by frontend malloc + return data -# 读 kind:kvspaceGetHead 从 x‥langtype attr key 填 h.langtype,body 从偏移 8 起。 +# Read kind via kvspaceGetHead (from inline attr); body starts at offset 8 with length h.body_len. def read_kind(kv, key, data): h = Head() if lib.kvspaceGetHead(kv, key.encode(), ctypes.byref(h)) != 0: @@ -127,8 +136,7 @@ def __enter__(self): def __exit__(self, *a): self.close() - # 写即构造:memcpy 固定头取 body_len/storetype/ptr,就地/新位置二选一原语, - # 往返回指针写 body。langtype 由调用方显式给(写 x‥langtype attr key)。 + # Write into returned body pointer; WriteNewPlace writes langtype with the header/attr region. def set(self, key, langtype, val): if len(val) < KVSPACE_XVALUE_HEAD_LEN: return @@ -139,8 +147,9 @@ def set(self, key, langtype, val): dst = U8P() err = ctypes.create_string_buffer(256) if lib.kvspaceWriteInPlace(self.kv, key.encode(), 1, bl, ctypes.byref(dst), err, 256) != 0: - lib.kvspaceWriteNewPlace(self.kv, key.encode(), hd.ptr, hd.storetype, - langtype.encode(), bl, bl, ctypes.byref(dst), err, 256) + if lib.kvspaceWriteNewPlace(self.kv, key.encode(), hd.ptr, hd.storetype, + bl, bl, langtype.encode(), ctypes.byref(dst), err, 256) != 0: + raise RuntimeError(f"write {key}: {err.value.decode(errors='replace')}") if bl > 0 and dst: ctypes.memmove(dst, ctypes.byref(buf, KVSPACE_XVALUE_HEAD_LEN), bl) diff --git a/tutorial-c/04_rust.rs b/tutorial-c/04_rust.rs index 73f2e06..6248d7e 100644 --- a/tutorial-c/04_rust.rs +++ b/tutorial-c/04_rust.rs @@ -1,11 +1,11 @@ // 04_rust — Rust FFI 直调 libkvspace(dispatch 前端),shm:// 后端 -// #566:head 固定 8 字节(body 偏移恒 8),类型走 x‥langtype attr key。 +// #571: Fixed 8-byte header (body offset is always 8); langtype in inline attr region. // 编译: rustc 04_rust.rs -l kvspace use std::ffi::{c_char, c_int, c_void, CString}; use std::ptr; -// 与 kvspaceHead_t 对齐(langtype 由 kvspaceGetHead 从 attr key 填)。 +// Matches kvspaceHead_t (langtype populated by kvspaceGetHead from inline attr). #[repr(C)] struct Head { headlen: u16, @@ -53,10 +53,9 @@ extern "C" { // 就地写:key 已存在且 body_len 不变 → 返回原 box body 偏移指针;否则非 0。 fn kvspaceWriteInPlace(h: *mut c_void, key: *const c_char, resolve: c_int, body_len: u32, body: *mut *mut u8, err: *mut c_char, err_cap: u32) -> c_int; - // 新位置写:按 (ref, storetype, langtype, body_len) 分配 box、写 head(顺写 attr key), - // 返回 body 偏移指针。 + // Allocate box and write fixed header + attr region (including langtype), returning body pointer. fn kvspaceWriteNewPlace(h: *mut c_void, key: *const c_char, r#ref: u8, storetype: u8, - langtype: *const c_char, body_len: u32, body_cap: u64, + body_len: u32, body_cap: u64, langtype: *const c_char, body: *mut *mut u8, err: *mut c_char, err_cap: u32) -> c_int; // 前缀枚举:ListLen 定计数,逐 idx ListAt 借用取名(不得 free)。 fn kvspaceListLen(h: *mut c_void, prefix: *const c_char, expand_ext: c_int, @@ -66,8 +65,10 @@ extern "C" { fn kvspaceDel(h: *mut c_void, keys: *const *const c_char, n: u32, err: *mut c_char, err_cap: u32) -> c_int; fn kvspaceDelTree(h: *mut c_void, prefix: *const c_char, err: *mut c_char, err_cap: u32) -> c_int; fn kvspaceMkindex(h: *mut c_void, path: *const c_char, capacity: u32, err: *mut c_char, err_cap: u32) -> c_int; - fn kvspaceNewInt64(v: i64, out: *mut *mut u8, out_len: *mut u32) -> c_int; - fn kvspaceNewChar(bytes: *const u8, len: u32, out: *mut *mut u8, out_len: *mut u32) -> c_int; + // New*: langtype is stored in inline attr (empty = untyped). + fn kvspaceNewInt64(v: i64, langtype: *const c_char, out: *mut *mut u8, out_len: *mut u32) -> c_int; + fn kvspaceNewChar(bytes: *const u8, len: u32, langtype: *const c_char, + out: *mut *mut u8, out_len: *mut u32) -> c_int; } fn cs(s: &str) -> CString { CString::new(s).unwrap() } @@ -75,7 +76,8 @@ fn cs(s: &str) -> CString { CString::new(s).unwrap() } fn enc_int(v: i64) -> Vec { let mut out: *mut u8 = ptr::null_mut(); let mut n: u32 = 0; - unsafe { kvspaceNewInt64(v, &mut out, &mut n); } + let lt = cs("int64"); + assert_eq!(unsafe { kvspaceNewInt64(v, lt.as_ptr(), &mut out, &mut n) }, 0); let b = unsafe { std::slice::from_raw_parts(out, n as usize) }.to_vec(); unsafe { free(out as *mut c_void); } b @@ -85,13 +87,14 @@ fn new_str(s: &str) -> Vec { let b = s.as_bytes(); let mut out: *mut u8 = ptr::null_mut(); let mut n: u32 = 0; - unsafe { kvspaceNewChar(b.as_ptr(), b.len() as u32, &mut out, &mut n); } + let lt = cs("char/utf8"); + assert_eq!(unsafe { kvspaceNewChar(b.as_ptr(), b.len() as u32, lt.as_ptr(), &mut out, &mut n) }, 0); let r = unsafe { std::slice::from_raw_parts(out, n as usize) }.to_vec(); unsafe { free(out as *mut c_void); } r } -// 读 kind:kvspaceGetHead 从 x‥langtype attr key 填 h.langtype,body 从偏移 8 起。 +// Read kind via kvspaceGetHead (from inline attr); body starts at offset 8 with length h.body_len. fn read_kind(kv: *mut c_void, key: &str, data: &[u8]) -> (String, Vec) { let mut h = Head::default(); if unsafe { kvspaceGetHead(kv, cs(key).as_ptr(), &mut h) } != 0 { @@ -116,8 +119,7 @@ impl KV { assert!(!ptr.is_null()); KV { ptr, path: path.to_string() } } - // 写即构造:memcpy 固定头取 body_len/storetype/ptr,就地/新位置二选一原语, - // 往返回指针写 body。langtype 由调用方显式给(写 x‥langtype attr key)。 + // Write into returned body pointer; WriteNewPlace writes langtype with the header/attr region. fn set(&self, key: &str, langtype: &str, val: &[u8]) { if val.len() < 8 { return; } let hd: XvalueHead = unsafe { std::ptr::read(val.as_ptr() as *const XvalueHead) }; @@ -128,8 +130,10 @@ impl KV { let mut err = [0i8; 256]; unsafe { if kvspaceWriteInPlace(self.ptr, ck.as_ptr(), 1, bl, &mut dst, err.as_mut_ptr(), 256) != 0 { - kvspaceWriteNewPlace(self.ptr, ck.as_ptr(), hd.ptr, hd.storetype, lt.as_ptr(), - bl, bl as u64, &mut dst, err.as_mut_ptr(), 256); + let rc = kvspaceWriteNewPlace(self.ptr, ck.as_ptr(), hd.ptr, hd.storetype, + bl, bl as u64, lt.as_ptr(), &mut dst, + err.as_mut_ptr(), 256); + assert_eq!(rc, 0, "write {key} failed"); } if bl > 0 && !dst.is_null() { ptr::copy_nonoverlapping(val[8..].as_ptr(), dst, bl as usize); diff --git a/tutorial-c/05_integrity.c b/tutorial-c/05_integrity.c index f8c69e1..0ab2e9e 100644 --- a/tutorial-c/05_integrity.c +++ b/tutorial-c/05_integrity.c @@ -18,7 +18,7 @@ static int check(void *kv, const char *key, int64_t expected, int round) { uint32_t len; uint8_t *v = kv_get(kv, key, &len); if (!v || len == 0) { printf("ROUND%d MISS %s\n", round, key); return 0; } char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, key, v, len, kind, sizeof(kind), &body, &bl); + read_kind(v, len, kind, sizeof(kind), &body, &bl); if (bl < 8 || strcmp(kind, "int64") != 0) { printf("ROUND%d CORRUPT %s: bad kind=%s bl=%u\n", round, key, kind, bl); free(v); return 0; @@ -36,7 +36,7 @@ static int check_str(void *kv, const char *key, const char *expected, int round) uint32_t len; uint8_t *v = kv_get(kv, key, &len); if (!v || len == 0) { printf("ROUND%d MISS %s\n", round, key); return 0; } char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, key, v, len, kind, sizeof(kind), &body, &bl); + read_kind(v, len, kind, sizeof(kind), &body, &bl); if (strcmp(kind, "char/utf8") != 0) { printf("ROUND%d CORRUPT %s: bad kind\n", round, key); free(v); return 0; } @@ -138,7 +138,7 @@ int main(void) { uint32_t len; uint8_t *v = kv_get(kv, k, &len); if (!v || len == 0) { printf("ROUND5 MISS big%d\n", i); errors++; continue; } char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, k, v, len, kind, sizeof(kind), &body, &bl); + read_kind(v, len, kind, sizeof(kind), &body, &bl); if (bl != MAX_VAL_SZ || memcmp(body, big_vals[i], MAX_VAL_SZ) != 0) { printf("ROUND5 CORRUPT big%d\n", i); errors++; } diff --git a/tutorial-c/06_multiprocess.c b/tutorial-c/06_multiprocess.c index 76d4741..c3db3b7 100644 --- a/tutorial-c/06_multiprocess.c +++ b/tutorial-c/06_multiprocess.c @@ -88,7 +88,7 @@ static int reader(const char *path, int64_t seed) { uint32_t l; uint8_t *v = kv_get(kv, k, &l); if (v && l > 0) { char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, k, v, l, kind, sizeof(kind), &body, &bl); + read_kind(v, l, kind, sizeof(kind), &body, &bl); if (bl >= 8 && strcmp(kind, "int64") == 0) { int64_t got = *(const int64_t *)body; if (got != expected) { @@ -108,7 +108,7 @@ static int reader(const char *path, int64_t seed) { uint32_t l; uint8_t *v = kv_get(kv, k, &l); if (!v || l == 0) continue; char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, k, v, l, kind, sizeof(kind), &body, &bl); + read_kind(v, l, kind, sizeof(kind), &body, &bl); if (strcmp(kind, "char/utf8") != 0) { printf("[reader] BADKIND %s: %s\n", k, kind); errors++; } @@ -123,7 +123,7 @@ static int reader(const char *path, int64_t seed) { uint32_t l; uint8_t *v = kv_get(kv, k, &l); if (v && l > 0) { char kind[32]; const uint8_t *body; uint32_t bl; - read_kind(kv, k, v, l, kind, sizeof(kind), &body, &bl); + read_kind(v, l, kind, sizeof(kind), &body, &bl); if (bl == 1024) big_ok++; free(v); } diff --git a/tutorial-c/common.h b/tutorial-c/common.h index f5df54c..c5d529e 100644 --- a/tutorial-c/common.h +++ b/tutorial-c/common.h @@ -1,6 +1,6 @@ // common.h — tutorial 公共助手:统一 kvspace* ABI 的 XValue 构造/读取与单键读写便捷封装。 // 所有 tutorial 均链接 libkvspace(dispatch 前端),运行期经 shm:// 选择 kvspace-c 后端。 -// #566:head 固定 8 字节(body 偏移恒 8),类型/形状走 x‥langtype attr key。 +// #571: Fixed 8-byte header (body offset is always 8); langtype (incl. [dims]) in inline attr. #ifndef KVSPACE_TUTORIAL_COMMON_H #define KVSPACE_TUTORIAL_COMMON_H @@ -11,51 +11,54 @@ #include #include -// ── 构造:返回 malloc 的 XValue 字节(固定头 + body),长度写入 *len,调用方 free ── +// Constructors: return malloc-allocated XValue bytes (header + body + attr); caller frees. static inline uint8_t *new_int64(int64_t v, uint32_t *len) { uint8_t *out = NULL; uint32_t n = 0; - kvspaceNewInt64(v, &out, &n); + kvspaceNewInt64(v, "int64", &out, &n); *len = n; return out; } static inline uint8_t *new_str(const char *s, uint32_t *len) { uint8_t *out = NULL; uint32_t n = 0; - kvspaceNewChar((const uint8_t *)s, (uint32_t)strlen(s), &out, &n); + kvspaceNewChar((const uint8_t *)s, (uint32_t)strlen(s), "char/utf8", &out, &n); *len = n; return out; } static inline uint8_t *new_bytes(const uint8_t *raw, uint32_t rl, uint32_t *len) { uint8_t *out = NULL; uint64_t n = 0; uint64_t dims[1] = { rl }; - kvspaceXvalueHeadNewTensor(dims, 1, raw, rl, &out, &n); + kvspaceXvalueHeadNewTensor(dims, 1, raw, rl, NULL, &out, &n); *len = n > UINT32_MAX ? 0 : (uint32_t)n; return out; } -// ── 读取:kvspaceGetHead 读 x‥langtype attr key 取 base kind(去 */@ 前缀与 [dims] 段), -// 返回 body 指针/长度(body 偏移恒 8)── +// Reader: bounded langtype parse from inline attr (stripping leading [dims] for base kind); +// body starts at offset 8 with length from header. -static inline const char *read_kind(void *h, const char *key, const uint8_t *data, uint32_t len, - char *kind, size_t cap, - const uint8_t **body, uint32_t *body_len) { - if (!h || !key || !data || len < KVSPACE_XVALUE_HEAD_LEN) return NULL; - kvspaceHead_t hd; - if (kvspaceGetHead(h, key, &hd) != 0) return NULL; - const char *k = (const char *)hd.langtype; - if (k[0] == '*' || k[0] == '@') k++; - const char *base = k; - if (k[0] == '[') { const char *e = strchr(k, ']'); if (e) base = e + 1; } - size_t n = strlen(base); +static inline const char *read_kind(const uint8_t *data, uint32_t len, + char *kind, size_t cap, + const uint8_t **body, uint32_t *body_len) { + if (cap) kind[0] = 0; + *body = NULL; *body_len = 0; + if (!data || len < KVSPACE_XVALUE_HEAD_LEN || cap == 0) return NULL; + kvspaceXvalueHead hd; + memcpy(&hd, data, sizeof hd); + uint64_t bl = kvspaceXvalueHeadLen(&hd); + if (bl > len - KVSPACE_XVALUE_HEAD_LEN) return NULL; + const uint8_t *lt = NULL; + size_t n = kvspaceXvalueHeadLangtypeBounded(data, len, <); + const uint8_t *e = lt && n && lt[0] == '[' ? memchr(lt, ']', n) : NULL; + if (e) { n -= (size_t)(e + 1 - lt); lt = e + 1; } if (n >= cap) n = cap - 1; - memcpy(kind, base, n); kind[n] = 0; + if (n) memcpy(kind, lt, n); + kind[n] = 0; *body = data + KVSPACE_XVALUE_HEAD_LEN; - *body_len = len - KVSPACE_XVALUE_HEAD_LEN; + *body_len = (uint32_t)bl; return kind; } -// ── 单键读写封装(写:New* 构造 XValue + 显式 langtype → WriteNewPlace 顺写 attr key+body; -// 就地/新位置二选一原语)── +// Single-key read/write helpers (WriteInPlace or WriteNewPlace with inline langtype). static inline int kv_set(void *h, const char *key, const char *langtype, const uint8_t *val, uint32_t len) { @@ -65,8 +68,8 @@ static inline int kv_set(void *h, const char *key, const char *langtype, uint64_t bl = kvspaceXvalueHeadLen(&hd); uint8_t *dst = NULL; char err[256]; if (kvspaceWriteInPlace(h, key, 1, (uint32_t)bl, &dst, err, sizeof err) != 0 && - kvspaceWriteNewPlace(h, key, hd.ptr, hd.storetype, langtype, - (uint32_t)bl, bl, &dst, err, sizeof err) != 0) + kvspaceWriteNewPlace(h, key, hd.ptr, hd.storetype, (uint32_t)bl, bl, + langtype, &dst, err, sizeof err) != 0) return -1; if (bl && dst) memcpy(dst, val + KVSPACE_XVALUE_HEAD_LEN, (size_t)bl); return 0; diff --git a/tutorial-c/test.py b/tutorial-c/test.py index a98344a..5be6f36 100644 --- a/tutorial-c/test.py +++ b/tutorial-c/test.py @@ -7,9 +7,10 @@ ROOT = Path(__file__).resolve().parent BUILD = ROOT.parent / "build" / "tutorial-c" -# #566:用仓库 include(固定 8 字节 head + New* 构造器);系统 /usr/include 仍是旧 wire 头。 +# Use repo include/ matching this frontend version rather than potentially stale /usr/include. INC = str(ROOT.parent / "include") -LIB = "/usr/lib" +# Frontend/backend library directory (/lib/kvspace from cmake --install; override via KVSPACE_LIB). +LIB = os.environ.get("KVSPACE_LIB", "/usr/lib/kvspace") passed = 0 failed = 0 diff --git a/tutorial-durable/01-basic.sh b/tutorial-durable/01-basic.sh index 1232284..0bea5b4 100644 --- a/tutorial-durable/01-basic.sh +++ b/tutorial-durable/01-basic.sh @@ -19,11 +19,11 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /t01/ echo "=== Set & Get ===" -$KV set /t01/ index: +$KV mkindex /t01/ $KV set /t01/a int:42 $KV get /t01/a diff --git a/tutorial-durable/02-link.sh b/tutorial-durable/02-link.sh index 2344bed..7a5a48c 100644 --- a/tutorial-durable/02-link.sh +++ b/tutorial-durable/02-link.sh @@ -24,14 +24,15 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /tgt/ +$KV deltree /lnk -$KV set /tgt/ index: +$KV mkindex /tgt/ $KV set /tgt/x int:42 $KV set /tgt/y int:7 $KV set /tgt/z string:hello -$KV set /lnk "*index:/tgt/" +$KV set /lnk "*:lib:/tgt/" echo "=== read through ===" $KV get /lnk/x /lnk/y diff --git a/tutorial-durable/03-extindex.sh b/tutorial-durable/03-extindex.sh index f1ecda5..a5c44a5 100644 --- a/tutorial-durable/03-extindex.sh +++ b/tutorial-durable/03-extindex.sh @@ -32,11 +32,11 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /base/ $KV deltree /merge/ -$KV set /base/ index: +$KV mkindex /base/ $KV set /base/a int:1 $KV set /base/b int:2 $KV set /base/c int:3 @@ -59,7 +59,7 @@ $KV get /merge/z $KV list /merge/ echo "=== nested write ===" -$KV set /merge/nest/ index: +$KV mkindex /merge/nest/ $KV set /merge/nest/x int:42 $KV set /merge/nest/y int:43 $KV get /merge/nest/x /merge/nest/y diff --git a/tutorial-durable/04-unlink.sh b/tutorial-durable/04-unlink.sh index f37cbad..357d463 100644 --- a/tutorial-durable/04-unlink.sh +++ b/tutorial-durable/04-unlink.sh @@ -10,21 +10,23 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /tgt/ $KV deltree /base/ +$KV deltree /merge/ +$KV deltree /lnk echo "=== unlink link ===" -$KV set /tgt/ index: +$KV mkindex /tgt/ $KV set /tgt/x int:1 -$KV set /lnk "*index:/tgt/" +$KV set /lnk "*:lib:/tgt/" $KV get /lnk/x $KV del /lnk $KV get /lnk $KV get /tgt/x echo "=== unlink extindex ===" -$KV set /base/ index: +$KV mkindex /base/ $KV set /base/a int:2 $KV extindex /merge/ /base/ $KV set /merge/b int:3 diff --git a/tutorial-durable/05-dir-value.sh b/tutorial-durable/05-dir-value.sh index c9b77e7..9043825 100644 --- a/tutorial-durable/05-dir-value.sh +++ b/tutorial-durable/05-dir-value.sh @@ -12,24 +12,24 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /dv/ $KV deltree /dv2/ echo "=== dir with value and child ===" -$KV set /dv/ index: +$KV mkindex /dv/ $KV set /dv int:99 $KV set /dv/x int:1 $KV get /dv $KV list /dv/ $KV get /dv/x -$KV set /dv/x/ index: +$KV mkindex /dv/x/ $KV set /dv/x/y int:2 $KV get /dv/x/y $KV list /dv/x/ echo "== del value keeps children ===" -$KV set /dv2/ index: +$KV mkindex /dv2/ $KV set /dv2/x int:42 $KV del /dv2 $KV get /dv2/x diff --git a/tutorial-durable/06-deep-nest.sh b/tutorial-durable/06-deep-nest.sh index 64222a5..cd02774 100644 --- a/tutorial-durable/06-deep-nest.sh +++ b/tutorial-durable/06-deep-nest.sh @@ -2,31 +2,30 @@ # expected: # === deep tree === # /deep/a/b/c/d/e/f int64:6 -# a/ index (1) +# a/ lib # /deep/a/b/c int64:3 -# d/ index (1) -# c int64 3 -# c/ index (1) +# d/ lib +# c/ lib # === deltree mid === # /deep/a/b/c/d/e/f (nil) # /deep/a/b/c/d/e (nil) # /deep/a/b/c int64:3 # === deltree leaf === # /deep/a/b/c (nil) -# a/ index (0) +# a/ lib # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /deep/ echo "=== deep tree ===" -$KV set /deep/ index: -$KV set /deep/a/ index: -$KV set /deep/a/b/ index: -$KV set /deep/a/b/c/ index: -$KV set /deep/a/b/c/d/ index: -$KV set /deep/a/b/c/d/e/ index: +$KV mkindex /deep/ +$KV mkindex /deep/a/ +$KV mkindex /deep/a/b/ +$KV mkindex /deep/a/b/c/ +$KV mkindex /deep/a/b/c/d/ +$KV mkindex /deep/a/b/c/d/e/ $KV set /deep/a/b/c/d/e/f int:6 $KV get /deep/a/b/c/d/e/f $KV list /deep/ diff --git a/tutorial-durable/07-type-variety.sh b/tutorial-durable/07-type-variety.sh index e268df7..3e62795 100644 --- a/tutorial-durable/07-type-variety.sh +++ b/tutorial-durable/07-type-variety.sh @@ -20,9 +20,9 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /t/ -$KV set /t/ index: +$KV mkindex /t/ echo "=== int ===" $KV set /t/zero int:0 diff --git a/tutorial-durable/08-edge-cases.sh b/tutorial-durable/08-edge-cases.sh index fd3e270..4a03fa0 100644 --- a/tutorial-durable/08-edge-cases.sh +++ b/tutorial-durable/08-edge-cases.sh @@ -10,9 +10,9 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /e/ -$KV set /e/ index: +$KV mkindex /e/ echo "=== overwrite kind ===" $KV set /e/v int:42 diff --git a/tutorial-durable/09-bulk-ops.sh b/tutorial-durable/09-bulk-ops.sh index dae919d..1c10b41 100644 --- a/tutorial-durable/09-bulk-ops.sh +++ b/tutorial-durable/09-bulk-ops.sh @@ -16,12 +16,12 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /b/ -$KV set /b/ index: -$KV set /b/a/ index: -$KV set /b/b/ index: +$KV mkindex /b/ +$KV mkindex /b/a/ +$KV mkindex /b/b/ echo "=== bulk get ===" $KV set /b/a/x int:1 @@ -30,7 +30,7 @@ $KV set /b/b/z int:3 $KV get /b/a/x /b/a/y /b/b/z echo "=== bulk del siblings ===" -$KV set /b/c/ index: +$KV mkindex /b/c/ $KV set /b/c/x int:1 $KV set /b/c/y int:2 $KV set /b/c/z int:3 @@ -38,9 +38,9 @@ $KV del /b/c/x /b/c/y $KV get /b/c/x /b/c/y /b/c/z echo "=== bulk del cross dirs ===" -$KV set /b/d/ index: -$KV set /b/e/ index: -$KV set /b/f/ index: +$KV mkindex /b/d/ +$KV mkindex /b/e/ +$KV mkindex /b/f/ $KV set /b/d/a int:4 $KV set /b/e/b int:5 $KV set /b/f/c int:6 diff --git a/tutorial-durable/10-dir-file-coexist.sh b/tutorial-durable/10-dir-file-coexist.sh index af614c3..70c2fa4 100644 --- a/tutorial-durable/10-dir-file-coexist.sh +++ b/tutorial-durable/10-dir-file-coexist.sh @@ -1,22 +1,28 @@ #!/bin/bash # expected: -# === tree shows dir and file === -# / -# ├── b value +# === dir and file coexist === +# /t10/b char/utf8:value +# /t10/b/x int64:1 +# === tree === +# /t10/ # └── b/ # └── x 1 # /end set -e -KV="$HOME/.local/bin/kvspace" -$KV deltree /b/ +KV="${KVSPACE_BIN:-kvspace}" +$KV deltree /t10/ -$KV set /b/ index: -$KV set /b string:value -$KV set /b/x int:1 +$KV mkindex /t10/ +$KV mkindex /t10/b/ +$KV set /t10/b string:value +$KV set /t10/b/x int:1 -echo "=== tree shows dir and file ===" -$KV tree --showext=false / +echo "=== dir and file coexist ===" +$KV get /t10/b /t10/b/x + +echo "=== tree ===" +$KV tree --showext=false /t10/ diff --git a/tutorial-durable/11-extindex-tree.sh b/tutorial-durable/11-extindex-tree.sh index 6c57208..6e2b49f 100644 --- a/tutorial-durable/11-extindex-tree.sh +++ b/tutorial-durable/11-extindex-tree.sh @@ -15,15 +15,15 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /ext/ $KV deltree /base/ -$KV set /base/ index: +$KV mkindex /base/ $KV set /base/a int:1 $KV extindex /ext/ /base/ $KV set /ext/z int:42 -$KV set /ext/x/ index: +$KV mkindex /ext/x/ $KV set /ext/x/z int:99 echo "=== tree showext true ===" diff --git a/tutorial-durable/12-extindex-dir-file.sh b/tutorial-durable/12-extindex-dir-file.sh index a64ee99..747c47d 100644 --- a/tutorial-durable/12-extindex-dir-file.sh +++ b/tutorial-durable/12-extindex-dir-file.sh @@ -1,25 +1,31 @@ #!/bin/bash # expected: +# === get dir value and child === +# /ext/d char/utf8:val +# /ext/d/y int64:2 +# /ext/k int64:1 # === tree shows extindex with dir+file === # /ext/ -# ├── d val # ├── d/ # │ └── y 2 # └── k 1 # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /ext/ $KV deltree /base/ -$KV set /base/ index: +$KV mkindex /base/ $KV set /base/k int:1 $KV extindex /ext/ /base/ -$KV set /ext/d/ index: +$KV mkindex /ext/d/ $KV set /ext/d string:val $KV set /ext/d/y int:2 +echo "=== get dir value and child ===" +$KV get /ext/d /ext/d/y /ext/k + echo "=== tree shows extindex with dir+file ===" $KV tree /ext/ diff --git a/tutorial-durable/13-extindex-cascade.sh b/tutorial-durable/13-extindex-cascade.sh index f7021d7..6c65e70 100644 --- a/tutorial-durable/13-extindex-cascade.sh +++ b/tutorial-durable/13-extindex-cascade.sh @@ -8,12 +8,12 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /ext/ $KV deltree /base/ $KV deltree /mid/ -$KV set /base/ index: +$KV mkindex /base/ $KV set /base/a int:1 $KV extindex /mid/ /base/ diff --git a/tutorial-durable/14-head-perm.sh b/tutorial-durable/14-head-perm.sh index 69ff9e8..3230613 100644 --- a/tutorial-durable/14-head-perm.sh +++ b/tutorial-durable/14-head-perm.sh @@ -1,31 +1,31 @@ #!/bin/bash # expected: -# === default (rw, vid 0) === -# /t14/a int64 ref=0 ro=0 vid=0 ndim=0 dims=[] -# === read-only (ro=1) === -# /t14/b int64 ref=0 ro=1 vid=0 ndim=0 dims=[] -# === ro + vid === -# /t14/c char/utf8 ref=0 ro=1 vid=7 ndim=1 dims=[5] -# === soft link (ref=1) + vid === -# /t14/d int64 ref=1 ro=0 vid=9 ndim=0 dims=[] +# === scalar (ref=0, storetype=0) === +# /t14/a int64 ref=0 storetype=0 ndim=0 dims=[] +# === slack string (ref=0, storetype=1) === +# /t14/c char/utf8 ref=0 storetype=1 ndim=0 dims=[] +# === typed ptr (ref=1, storetype=1) === +# /t14/d int64 ref=1 storetype=1 ndim=0 dims=[] +# === untyped ptr (ref=1, storetype=1) === +# /t14/e ref=1 storetype=1 ndim=0 dims=[] # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /t14/ -echo "=== default (rw, vid 0) ===" +echo "=== scalar (ref=0, storetype=0) ===" $KV set /t14/a int:42 $KV head /t14/a -echo "=== read-only (ro=1) ===" -$KV set /t14/b int:7 ro -$KV head /t14/b - -echo "=== ro + vid ===" -$KV set /t14/c string:hello ro 7 +echo "=== slack string (ref=0, storetype=1) ===" +$KV set /t14/c string:hello $KV head /t14/c -echo "=== soft link (ref=1) + vid ===" -$KV set /t14/d '*int64:/target' rw 9 +echo "=== typed ptr (ref=1, storetype=1) ===" +$KV set /t14/d '*:int64:/t14/a' $KV head /t14/d + +echo "=== untyped ptr (ref=1, storetype=1) ===" +$KV set /t14/e '*:/t14/a' +$KV head /t14/e diff --git a/tutorial-durable/15-map-coord.sh b/tutorial-durable/15-map-coord.sh index 8c48e00..b72dce1 100755 --- a/tutorial-durable/15-map-coord.sh +++ b/tutorial-durable/15-map-coord.sh @@ -1,7 +1,7 @@ #!/bin/bash # expected: # === map head === -# /m stringkeymap ref=0 ro=0 vid=0 ndim=2 dims=[2,3] +# /m [int64,int64]·float32 ref=0 storetype=0 ndim=0 dims=[] # === map list === # [0,0] float32 1.0 # [0,1] float32 3.140000104904175 @@ -15,12 +15,12 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /m/ $KV deltree /h/ echo "=== map head ===" -$KV set /m· 'map[2,3]:' +$KV set /m 'map:[int64,int64]·float32' $KV head /m echo "=== map list ===" @@ -33,7 +33,7 @@ echo "=== get ===" $KV get '/m·[1,2]' '/m·[9,9]' echo "=== 命名成员 vs 坐标成员 ===" -$KV set /h· 'map[0]:' # 显式声明容器(memhead),并建 memindex;object kind 已移除 +$KV set /h 'map:[]char/utf8·int64' # declare container (memhead) explicitly $KV set /h·x 'int:1' $KV set '/h·[0]' 'int:2' $KV list /h· --kind=false --showext=false diff --git a/tutorial-durable/16-geo-coord.sh b/tutorial-durable/16-geo-coord.sh index 96a40ac..1ab2e08 100644 --- a/tutorial-durable/16-geo-coord.sh +++ b/tutorial-durable/16-geo-coord.sh @@ -10,11 +10,11 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /geo/ echo "=== geo list ===" -$KV set /geo· 'map[0]:' # 先声明容器(memhead),否则成员写被拒 +$KV set /geo 'map:[float64,float64]·[]char/utf8' # declare container (memhead) before member writes $KV set '/geo·[39.9,116.4]' 'string:Beijing' $KV set '/geo·[31.23,121.47]' 'string:Shanghai' $KV list /geo· --kind --showext=false diff --git a/tutorial-durable/17-cp.sh b/tutorial-durable/17-cp.sh index d3f304f..a96997c 100644 --- a/tutorial-durable/17-cp.sh +++ b/tutorial-durable/17-cp.sh @@ -16,9 +16,11 @@ # /end set -e -KV="$HOME/.local/bin/kvspace" +KV="${KVSPACE_BIN:-kvspace}" $KV deltree /src/ $KV deltree /dst/ +$KV deltree /p/ +$KV deltree /q/ $KV deltree /d1/ $KV deltree /d2/ $KV deltree /base/ @@ -33,7 +35,7 @@ $KV get /dst/one # cpdir 复制 · 成员容器(base + memindex + 成员) echo "=== cpdir memindex container ===" -$KV set /p· 'map[0]:' # 容器值(memhead)+ memindex;index: 只建 memindex,成员写会被拒 +$KV set /p 'map:[]char/utf8·float64' # container value (memhead) required for `·` members $KV set /p·x float:3.14 $KV set /p·y int:42 $KV cpdir /p /q @@ -41,7 +43,7 @@ $KV list "/q·" # cpdir 复制 /-children 子树 echo "=== cpdir /-children subtree ===" -$KV set /d1/ index: +$KV mkindex /d1/ $KV set /d1/a int:1 $KV set /d1/b int:2 $KV cpdir /d1 /d2 @@ -49,10 +51,10 @@ $KV get /d2/a /d2/b # cpdir 遇 extindex:target 侧生成指向同一只读扩展的新 extindex echo "=== cpdir extindex → new overlay on target ===" -$KV set /realext/ index: +$KV mkindex /realext/ $KV set /realext/a int:1 $KV set /realext/b int:2 -$KV set /base/ index: +$KV mkindex /base/ $KV extindex /base/ /realext/ $KV cpdir /base /ov $KV get /ov/a /ov/b diff --git a/tutorial-durable/test.py b/tutorial-durable/test.py index 86e6533..cb8875f 100644 --- a/tutorial-durable/test.py +++ b/tutorial-durable/test.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""运行 tutorial/*.sh,对比脚本头部注释中的 expected 输出; -交叉校验 kvspace-c 与 kvspace-durable 的 head 编解码(rw/vid)字节一致; -并回归写侧落盘时机:下一次 op 落盘与退出兜底(issue #23)。""" +"""Run tutorial-durable/*.sh and compare against expected header comments; +cross-check XValue constructors (New*) and header decoding (GetHead) between +kvspace-c and kvspace-durable; and verify write flush on next op and exit (issue #23).""" import ctypes import os +import shutil import struct import subprocess import sys @@ -14,10 +15,38 @@ ROOT = Path(__file__).resolve().parent.parent # kvspace/ WS = ROOT.parent # array2d/ -DURABLE_SO = WS / "kvspace-durable" / "target" / "release" / "libkvspace_durable.so" + + +def find_first(env_key, candidates): + override = os.environ.get(env_key) + if override: + return Path(override) + for c in candidates: + p = Path(c) + if p.exists(): + return p + return Path(candidates[0]) + + +DURABLE_SO = find_first("KVSPACE_DURABLE_SO", [ + WS / "kvspace-durable" / "target" / "release" / "libkvspace_durable.so", + "/usr/lib/kvspace/libkvspace_durable.so.1", +]) KVSPACE_C_DIR = WS / "kvspace-c" -KVSPACE_C_SO = KVSPACE_C_DIR / "build" / "libkvspace-c.so" -FRONTEND_SO = os.environ.get("KVSPACE_SO", "/usr/lib/kvspace/libkvspace.so.1") +KVSPACE_C_SO = find_first("KVSPACE_C_SO", [ + KVSPACE_C_DIR / "build" / "libkvspace-c.so", + "/usr/lib/kvspace/libkvspace-c.so.1", +]) +FRONTEND_SO = str(find_first("KVSPACE_SO", [ + ROOT / "build" / "libkvspace.so.1", + "/usr/lib/kvspace/libkvspace.so.1", +])) +KVSPACE_BIN = str(find_first("KVSPACE_BIN", [ + ROOT / "cli" / "target" / "release" / "kvspace", + os.path.expanduser("~/.local/bin/kvspace"), + shutil.which("kvspace") or "/usr/bin/kvspace", +])) + def extract_expected(script): """从脚本头部 # expected: ... # /end 提取预期输出行。""" @@ -37,132 +66,211 @@ def extract_expected(script): in_block = True return lines + def run_script(script): - """先 clear,再执行脚本,返回 stdout 行列表。""" - kvbin = os.path.expanduser('~/.local/bin/kvspace') + """Run script (each cleans up its own prefix via deltree without clearing shared redis) and return stdout lines.""" env = os.environ.copy() - env.setdefault('KVSPACE', 'redis://127.0.0.1:6379') - subprocess.run([kvbin, 'clear'], capture_output=True, timeout=10, env=env) + env['KVSPACE_BIN'] = KVSPACE_BIN + if 'KVSPACE' not in env: + with tempfile.TemporaryDirectory(prefix='kvspace-tut-') as tmp: + env['KVSPACE'] = f'fs://{tmp}/store' + r = subprocess.run(['bash', script], capture_output=True, text=True, timeout=30, env=env) + return r.stdout.rstrip('\n').split('\n') if r.stdout.strip() else [], r + # Safe to clear explicit fs:// paths; redis:// relies on per-script deltree prefix cleanup. + if env['KVSPACE'].startswith('fs://'): + subprocess.run([KVSPACE_BIN, 'clear'], capture_output=True, timeout=10, env=env) r = subprocess.run(['bash', script], capture_output=True, text=True, timeout=30, env=env) - return r.stdout.rstrip('\n').split('\n') if r.stdout.strip() else [] + return (r.stdout.rstrip('\n').split('\n') if r.stdout.strip() else []), r + def test_script(script): expected = extract_expected(script) - actual = run_script(script) - if expected == actual: + actual, r = run_script(script) + if r.returncode == 0 and expected == actual: print(f'PASS {os.path.basename(script)}') return True - print(f'FAIL {os.path.basename(script)}') + print(f'FAIL {os.path.basename(script)} (rc={r.returncode})') print(f' expected ({len(expected)} lines): {expected[:3]}...' if len(expected) > 3 else f' expected: {expected}') print(f' actual ({len(actual)} lines): {actual[:3]}...' if len(actual) > 3 else f' actual: {actual}') + if r.stderr.strip(): + print(f' stderr: {r.stderr.strip()[:300]}') return False -# ── kvspace-c ↔ kvspace-durable 交叉校验(ctypes,字节级对齐) ────────────── +# -- kvspace-c <-> kvspace-durable cross-check (ctypes, #571 v0.3.23+ ABI byte alignment) -- class HeadV(ctypes.Structure): - # kvspaceHead_t 的**解码视图**(对齐 kvspace/include/kvspace/kvspace.h)。 - # 注意:wire 上没有独立的 ref / storetype 字节——它们是 flags 字节解码后的派生视图 - #(低 2 位 storage class、bit 2 指针位),此处字段仅为跨边界读取元数据方便。 + # Matches kvspaceHead_t field layout in kvspace/include/kvspace/kvspace.h. _fields_ = [ ("headlen", ctypes.c_uint16), ("ref", ctypes.c_uint8), ("storetype", ctypes.c_uint8), - ("ro", ctypes.c_uint8), - ("vid", ctypes.c_uint32), ("body_len", ctypes.c_int32), ("ndim", ctypes.c_int32), ("dims", ctypes.c_int32 * 8), - ("langtype", ctypes.c_uint8 * 256), + ("langtype", ctypes.c_char * 256), ("langtype_len", ctypes.c_int32), ("body_offset", ctypes.c_int32), + ("body_cap", ctypes.c_uint64), ] U8P = ctypes.POINTER(ctypes.c_uint8) -I32P = ctypes.POINTER(ctypes.c_int32) U32P = ctypes.POINTER(ctypes.c_uint32) U8PP = ctypes.POINTER(U8P) -# codec 产出为 frontend malloc 缓冲,拷出后以 libc free 释放(无 kvspaceBytesFree)。 -_libc = ctypes.CDLL(None); _libc.free.argtypes = [ctypes.c_void_p] +_libc = ctypes.CDLL(None) +_libc.free.argtypes = [ctypes.c_void_p] def setup_lib(lib): - lib.kvspaceXvalueEncodeMode.argtypes = [ctypes.c_char_p, U8P, ctypes.c_uint32, I32P, ctypes.c_int32, - ctypes.c_int, ctypes.c_uint8, ctypes.c_uint32, U8PP, U32P] - lib.kvspaceXvalueEncodeMode.restype = ctypes.c_int - lib.kvspaceDecodeHead.argtypes = [U8P, ctypes.c_uint32, ctypes.POINTER(HeadV)] - lib.kvspaceDecodeHead.restype = ctypes.c_int - - -def encode(lib, kind, raw, dims=(), ref=0, ro=0, vid=0): - raw_arr = (ctypes.c_uint8 * len(raw)).from_buffer_copy(raw) - ndim = len(dims) - dims_arr = (ctypes.c_int32 * ndim)(*dims) if ndim > 0 else None + lib.kvspaceConnect.argtypes = [ctypes.c_char_p] + lib.kvspaceConnect.restype = ctypes.c_void_p + lib.kvspaceClose.argtypes = [ctypes.c_void_p] + lib.kvspaceClose.restype = None + lib.kvspaceSetValue.argtypes = [ctypes.c_void_p, ctypes.c_char_p, U8P, ctypes.c_uint32, + ctypes.c_char_p, ctypes.c_uint32] + lib.kvspaceSetValue.restype = ctypes.c_int + lib.kvspaceGet.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, U8PP, U32P] + lib.kvspaceGet.restype = ctypes.c_int + lib.kvspaceGetHead.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(HeadV)] + lib.kvspaceGetHead.restype = ctypes.c_int + lib.kvspaceWriteNewPlace.argtypes = [ + ctypes.c_void_p, ctypes.c_char_p, ctypes.c_uint8, ctypes.c_uint8, + ctypes.c_uint32, ctypes.c_uint64, ctypes.c_char_p, + U8PP, ctypes.c_char_p, ctypes.c_uint32, + ] + lib.kvspaceWriteNewPlace.restype = ctypes.c_int + lib.kvspaceNewInt64.argtypes = [ctypes.c_int64, ctypes.c_char_p, U8PP, U32P] + lib.kvspaceNewInt64.restype = ctypes.c_int + lib.kvspaceNewFloat64.argtypes = [ctypes.c_double, ctypes.c_char_p, U8PP, U32P] + lib.kvspaceNewFloat64.restype = ctypes.c_int + lib.kvspaceNewBool.argtypes = [ctypes.c_uint8, ctypes.c_char_p, U8PP, U32P] + lib.kvspaceNewBool.restype = ctypes.c_int + lib.kvspaceNewChar.argtypes = [U8P, ctypes.c_uint32, ctypes.c_char_p, U8PP, U32P] + lib.kvspaceNewChar.restype = ctypes.c_int + lib.kvspaceNewPtr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, U8PP, U32P] + lib.kvspaceNewPtr.restype = ctypes.c_int + + +def call_new(lib, kind, val, langtype): out = U8P() out_len = ctypes.c_uint32() - rc = lib.kvspaceXvalueEncodeMode(kind.encode(), raw_arr, len(raw), dims_arr, ndim, - ref, ro, vid, ctypes.byref(out), ctypes.byref(out_len)) - assert rc == 0, f"encode({kind}) rc={rc}" + lt = None if langtype is None else langtype.encode() + if kind == "int64": + rc = lib.kvspaceNewInt64(val, lt, ctypes.byref(out), ctypes.byref(out_len)) + elif kind == "float64": + rc = lib.kvspaceNewFloat64(val, lt, ctypes.byref(out), ctypes.byref(out_len)) + elif kind == "bool": + rc = lib.kvspaceNewBool(val, lt, ctypes.byref(out), ctypes.byref(out_len)) + elif kind == "char": + arr = (ctypes.c_uint8 * len(val)).from_buffer_copy(val) if val else None + rc = lib.kvspaceNewChar(arr, len(val), lt, ctypes.byref(out), ctypes.byref(out_len)) + elif kind == "ptr": + rc = lib.kvspaceNewPtr(lt, val.encode(), ctypes.byref(out), ctypes.byref(out_len)) + else: + raise ValueError(kind) + assert rc == 0, f"New({kind}, {langtype}) rc={rc}" data = ctypes.string_at(out, out_len.value) - _libc.free(ctypes.cast(out, ctypes.c_void_p)) + # Only frontend and C outputs use libc malloc; avoid calling libc free on Rust cdylib Box<[u8]>. return data -def decode(lib, data): - buf = (ctypes.c_uint8 * len(data)).from_buffer_copy(data) - h = HeadV() - rc = lib.kvspaceDecodeHead(buf, len(data), ctypes.byref(h)) - assert rc == 0, f"decode rc={rc}" - return { - "headlen": h.headlen, "ref": h.ref, "storetype": h.storetype, - "langtype": bytes(h.langtype).split(b"\0", 1)[0].decode(), - "ndim": h.ndim, "dims": list(h.dims)[:h.ndim], - "ro": h.ro, "vid": h.vid, - "body_len": h.body_len, "body_offset": h.body_offset, +def put_and_head(lib, h, key, ref, storetype, raw, langtype): + err = ctypes.create_string_buffer(256) + body = U8P() + lt = langtype.encode() if langtype else b"" + rc = lib.kvspaceWriteNewPlace(h, key.encode(), ref, storetype, len(raw), len(raw), + lt, ctypes.byref(body), err, 256) + assert rc == 0, f"WriteNewPlace({key}) rc={rc}: {err.value}" + if raw: + ctypes.memmove(body, raw, len(raw)) + out, out_len = U8P(), ctypes.c_uint32() + assert lib.kvspaceGet(h, key.encode(), 0, ctypes.byref(out), ctypes.byref(out_len)) == 0 + wire = ctypes.string_at(out, out_len.value) + hd = HeadV() + assert lib.kvspaceGetHead(h, key.encode(), ctypes.byref(hd)) == 0 + meta = { + "headlen": hd.headlen, + "ref": hd.ref, + "storetype": hd.storetype, + "body_len": hd.body_len, + "langtype": bytes(hd.langtype).split(b"\0", 1)[0].decode(), + "langtype_len": hd.langtype_len, + "body_offset": hd.body_offset, } + return wire, meta def test_kvspace_c_alignment(): - """kvspace-c 与 kvspace-durable 对同一 head 输入产出字节级一致的结果。""" + """Verify kvspace-c and kvspace-durable produce byte-identical wire values and HeadV metadata.""" if not DURABLE_SO.exists(): - subprocess.run(["cargo", "build"], cwd=ROOT, check=True, capture_output=True) + subprocess.run(["cargo", "build", "--release"], cwd=WS / "kvspace-durable", + check=True, capture_output=True) if not KVSPACE_C_SO.exists(): - subprocess.run(["make", "kvspace-c"], cwd=KVSPACE_C_DIR / "build", check=True, capture_output=True) - - # kvspace-c 依赖 blockmalloc/slotsboxmalloc,先按绝对路径预加载(RTLD_GLOBAL), - # 让后续 dlopen kvspace-c 时按 SONAME 命中已加载对象。 - for dep in ["blockmalloc/build/libblockmalloc.so.1", "slotsboxmalloc/build/libslotsboxmalloc.so"]: - ctypes.CDLL(str(KVSPACE_C_DIR.parent / dep), mode=ctypes.RTLD_GLOBAL) + subprocess.run(["make", "kvspace-c"], cwd=KVSPACE_C_DIR / "build", + check=True, capture_output=True) lib_d = ctypes.CDLL(str(DURABLE_SO)) - lib_c = ctypes.CDLL(str(KVSPACE_C_SO)) + try: + lib_c = ctypes.CDLL(str(KVSPACE_C_SO)) + except OSError: + # Preload local shared blockmalloc/slotsboxmalloc only when dynamically linked. + for dep in ["blockmalloc/build/libblockmalloc.so.1", "slotsboxmalloc/build/libslotsboxmalloc.so"]: + dep_path = KVSPACE_C_DIR.parent / dep + if dep_path.exists(): + ctypes.CDLL(str(dep_path), mode=ctypes.RTLD_GLOBAL) + lib_c = ctypes.CDLL(str(KVSPACE_C_SO)) setup_lib(lib_d) setup_lib(lib_c) - cases = [ - ("int64", struct.pack("