From 75246c8df45ea93b5842c69b9ecdf4524035befc Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Fri, 30 Oct 2026 16:58:32 +0000 Subject: [PATCH 01/12] kvspace-durable: fix cache subtree invalidation, attr bounds check, and v0.3.23 New* ABI - ffi: invalidate the full subtree in DelTree/CpTree/CpList (normalizing trailing slash or member separator) - headlenpow: bound langtype_from_wire to data.len() so truncated or malformed attr trailers return None instead of panicking across C ABI - ffi: align kvspaceNew* signatures and wire output with kvspace.h v0.3.23 (write langtype verbatim to attr; reject empty NewPtr target) - remove unused xvalue/backend helpers and update README wire format docs --- README.md | 22 +++++--- src/backend.rs | 37 +++--------- src/const.rs | 1 - src/ffi.rs | 120 ++++++++++++++++++++++++++++++--------- src/fs/kvspace.rs | 12 ++-- src/headlenpow.rs | 42 +++++--------- src/kvspace_common.rs | 99 +------------------------------- src/store.rs | 2 +- src/xvalue.rs | 128 +++++++++--------------------------------- src/xvalue_byte.rs | 33 ----------- tests/metadata_fs.rs | 2 +- tests/prefix_list.rs | 1 - 12 files changed, 165 insertions(+), 334 deletions(-) diff --git a/README.md b/README.md index 19d2b61..7a2ed05 100644 --- a/README.md +++ b/README.md @@ -25,24 +25,28 @@ Crate types: `rlib`, `staticlib`, `cdylib` (`libkvspace_durable.so`). C ABI exported from the cdylib (`src/ffi.rs`): - lifecycle: `kvspaceConnect`, `kvspaceClose` -- KV read/write: `kvspaceGet` (borrow), synchronous `kvspaceSetValue`, `kvspaceWriteInPlace`, `kvspaceWriteNewPlace` -- enumerate / delete / copy: `kvspaceListLen`, `kvspaceListAt`, `kvspaceDel`, `kvspaceDelTree`, `kvspaceCp`, `kvspaceCpTree` +- KV read/write: `kvspaceGet` (borrow), `kvspaceGetPart`, `kvspaceGetHead`, synchronous `kvspaceSetValue`, `kvspaceSetPart`, `kvspaceWriteInPlace`, `kvspaceWriteNewPlace` +- enumerate / delete / copy: `kvspaceListLen`, `kvspaceListAt`, `kvspaceDel`, `kvspaceDelTree`, `kvspaceCp`, `kvspaceCpTree`, `kvspaceCpList` - directories / extindex: `kvspaceMkindex`, `kvspaceMkindexExt`, `kvspaceRmindexExt` - watch / clear: `kvspaceWatch`, `kvspaceClear` -- XValue codec (frontend malloc, caller `free()`s): `kvspaceXvalueEncode`, `kvspaceXvalueEncodeMode`, `kvspaceDecodeHead`, `kvspaceNewPtr`, `kvspaceNewChar`, `kvspaceNewBool`, `kvspaceNewInt64`, `kvspaceNewFloat64` +- XValue constructors (caller `free()`s; same signatures and bytes as the frontend and `kvspace-c`, `langtype` is written verbatim): `kvspaceNewPtr`, `kvspaceNewChar`, `kvspaceNewBool`, `kvspaceNewInt64`, `kvspaceNewFloat64` +- shm-only views, always `-2` here (callers fall back to the calls above): `kvspaceXvalueBodyView`, `kvspaceView*` The same ABI is implemented by `kvspace-c` (`shm://`), so a consumer (e.g. the kvlang layout) switches backends by DSN only, with no code change. ## XValue -The headlenpow wire format is byte-identical across this backend, `kvspace-c`, and the `kvspace` frontend. Its 18-byte prefix is `[headlenpow:u8][flags:u8][a:u64 LE][b:u64 LE]`. The UTF-8 langtype occupies the rest of the `2^headlenpow` head, followed by the body. `flags` uses its low two bits for the storage class and bit 2 for a pointer. +The wire format is byte-identical across this backend, `kvspace-c`, and the `kvspace` frontend (`kvspace/xvalue_head.h`), and a value is always one key: -- Class 0 stores short fixed values, `None`, maps, structs, and directory markers. `None` has an empty langtype; map members and directory children are enumerated from physical key prefixes. -- Class 1 stores a byte length and reserved capacity (`a`, `b`) for strings and code slots. UTF-8 records the code point count in its langtype and the byte count in `a`. -- Class 2 stores tensor element count and width. Its langtype carries the dimensions. -- Class 3 stores an external locator, with the target's actual langtype. Pointer values use class 1 with the pointer bit set and a target key in the body. +``` +[len:u48 LE @0-5][storetype:u8 @6][ptr:u8 @7][body: len bytes @8][attr] +attr = [attr_len:u16 LE (includes itself)][record...] empty attr = [2,0] +record = [tag:u8][len:u16 LE][data] tag 1 = langtype, 2+ reserved +``` -`ro` and `vid` live at `/.kvspace-meta/` as a separate XValue. Writes and copies preserve the sidecar; its absence means `ro=false, vid=0`. +- `storetype` 0 short fixed values, `None`, maps, structs and directory markers; 1 variable length (strings, code slots, pointer paths); 2 tensors (the langtype carries the dimensions); 3 external locators (`@ext`). +- `ptr` 0 inline, 1 pointer (body = target key, langtype = target kindexpr), 2 `@ext` (body = locator). +- An empty langtype means `None` for inline values. Map members and directory children are enumerated from physical key prefixes. ## Tutorial diff --git a/src/backend.rs b/src/backend.rs index 3547bc5..aa714fa 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -11,7 +11,8 @@ use crate::kvspace_common::{ use crate::r#const::*; use crate::store::KVStore; use crate::xvalue::{ - from_wire, from_wire_head, is_ptr, ptr_target, XValue, REF_EXT, REF_PTR, + from_wire, from_wire_head, is_ptr, ptr_target, XValue, REF_EXT, REF_INLINE, REF_PTR, ST_EXT, + ST_SHORT, }; pub struct Backend { @@ -120,7 +121,7 @@ impl Backend { let end = rest.find(['/', '·']).unwrap_or(rest.len()); if end > 0 { let name = &rest[..end]; - // RUNTIME_MEMBER_SEP（‥）前缀键是系统变量/langtype attr key，List 时隐藏。 + // Hide runtime system keys containing RUNTIME_MEMBER_SEP (‥) from List. if (dir != PATH_SEP || name != META_ROOT_NAME) && !name.contains(RUNTIME_MEMBER_SEP) { let direct_dir = @@ -403,25 +404,6 @@ impl KVSpace for Backend { result } - fn list_len(&mut self, prefix: &str, expand_ext: bool, resolve: bool) -> i32 { - self.list(prefix, expand_ext, resolve).len() as i32 - } - - fn list_at( - &mut self, - prefix: &str, - idx: i32, - expand_ext: bool, - resolve: bool, - ) -> Option { - if idx < 0 { - return None; - } - self.list(prefix, expand_ext, resolve) - .into_iter() - .nth(idx as usize) - } - fn del(&mut self, keys: &[String]) -> Result<(), String> { for key in keys { if crate::r#const::is_reserved(key) { @@ -497,7 +479,7 @@ impl KVSpace for Backend { if raw.is_empty() { return Err(format!("Cp: source not found: {}", src)); } - // attr key 与值同址：读解析后的物理 src 路径的类型。 + // Copy raw wire bytes verbatim (langtype lives in the value's attr trailer). let v = from_wire(&raw); self.set(&[KVPair { key: dst.to_string(), @@ -583,9 +565,8 @@ impl KVSpace for Backend { let _ = self.del_tree(&dst_base); for k in &keys { let suffix = &k[src_base.len()..]; - // Copy the base（含其 langtype attr key）、直接 · 成员（含成员 attr key）与 ext 目录。 + // Copy base value, direct '·' members, and ext directories (attrs travel with values). let one_level = suffix.is_empty() - || suffix == format!("{RUNTIME_MEMBER_SEP}langtype") || (suffix.starts_with(OBJ_SEP) && { let rest = &suffix[OBJ_SEP.len()..]; let member = rest.strip_suffix(PATH_SEP).unwrap_or(rest); @@ -612,8 +593,8 @@ impl KVSpace for Backend { } let _ = capacity; let resolved = self.resolve_path(path); - // 固定头空目录 marker：storetype 0、ptr 0、空 body；类型 "lib" 在值 attr 区。 - let value = crate::headlenpow::to_wire(0, 0, &[], "lib") + // Directory marker: ST_SHORT, inline, empty body, langtype "lib" in attr. + let value = crate::headlenpow::to_wire(ST_SHORT, REF_INLINE, &[], "lib") .ok_or_else(|| format!("Mkindex: invalid directory {resolved}"))?; if self.store.get(&resolved).is_none() { self.store.set(&resolved, &value); @@ -637,8 +618,8 @@ impl KVSpace for Backend { return Err(format!("ExtIndex target missing: {ext_path}")); } let langtype = self.langtype_of(ext_path); - // 扩展定位：storetype 3（ST_EXT）+ ptr 2（PTR_EXT）+ 定位串 body + 类型 attr 区。 - let value = crate::headlenpow::to_wire(3, 2, ext_path.as_bytes(), &langtype) + // External locator: ST_EXT + REF_EXT + locator body + target langtype in attr. + let value = crate::headlenpow::to_wire(ST_EXT, REF_EXT, ext_path.as_bytes(), &langtype) .ok_or_else(|| format!("ExtIndex locator invalid: {ext_path}"))?; self.store.set(&resolved, &value); Ok(()) diff --git a/src/const.rs b/src/const.rs index 2d0ff5e..403bea9 100644 --- a/src/const.rs +++ b/src/const.rs @@ -7,7 +7,6 @@ pub const PATH_SEP: &str = "/"; // 路径分隔符 pub const META_ROOT_NAME: &str = ".kvspace-meta"; // 保留区（不再有 meta sidecar，仅守卫防业务写） pub const DIR_INDEX_SUF: &str = "/"; // 目录索引键后缀（尾斜杠 = 目录，必须以 / 开头的 key 保证不冲突） pub const RUNTIME_MEMBER_SEP: &str = "‥"; // 运行时保留字段前缀（U+2025）——‥ 的唯一定义处，List 时隐藏 -pub const EXT_INDEX_HEAD: &str = "…"; // Directory display marker. /// `/.kvspace-meta` 保留区判定（原 metadata::is_reserved，随 meta 删除迁至常量模块）。 pub fn is_reserved(key: &str) -> bool { diff --git a/src/ffi.rs b/src/ffi.rs index 7dedc6c..34d2176 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -20,11 +20,8 @@ use std::time::Duration; use crate::conn::conn; use crate::kvspace::{KVPair, KVSpace}; -use crate::xvalue::{from_wire, from_wire_head, new_ptr}; -use crate::xvalue_bool::new_bool; -use crate::xvalue_byte::new_char_byte; -use crate::xvalue_float::new_float64; -use crate::xvalue_int::new_int64; +use crate::r#const::{DIR_INDEX_SUF, OBJ_SEP, PATH_SEP, RUNTIME_MEMBER_SEP}; +use crate::xvalue::{from_wire, from_wire_head, REF_INLINE, REF_PTR, ST_SHORT, ST_SLACK}; // ── 句柄 ───────────────────────────────────────────────────────────── // @@ -85,17 +82,32 @@ impl Handle { } } -/// key 是否等于 prefix 或位于其下（`/` 或 `‥` 分隔）。 +/// Returns whether `key` falls within the subtree affected by `DelTree`/`CpTree`/`CpList(prefix)`, +/// normalizing a trailing `/` or `·` to `base` and matching `/`, `·`, and `‥` boundaries. fn under_prefix(key: &str, prefix: &str) -> bool { if key == prefix { return true; } - if prefix.ends_with('/') || prefix.ends_with('\u{2025}') { + if prefix.ends_with(RUNTIME_MEMBER_SEP) { return key.starts_with(prefix); } - key.strip_prefix(prefix) - .map(|rest| rest.starts_with('/') || rest.starts_with('\u{2025}')) - .unwrap_or(false) + let base = if prefix == PATH_SEP { + "" + } else { + prefix + .strip_suffix(DIR_INDEX_SUF) + .or_else(|| prefix.strip_suffix(OBJ_SEP)) + .unwrap_or(prefix) + }; + if base.is_empty() { + return key.starts_with(PATH_SEP); + } + key.strip_prefix(base).is_some_and(|rest| { + rest.is_empty() + || rest.starts_with(PATH_SEP) + || rest.starts_with(OBJ_SEP) + || rest.starts_with(RUNTIME_MEMBER_SEP) + }) } /// 控制键重读后端并写入双缓冲的另一块（上一块保活到该 key 下一次读）。 @@ -980,9 +992,31 @@ pub extern "C" fn kvspaceClear(h: *mut Handle, err: *mut c_char, err_cap: u32) - code } -// ── XValue 编解码（head + 标准标量构造器） ───────────────────────────── - // ── 标准标量构造器（返回完整 XValue 字节） ───────────────────────────── +// +// Signatures and wire output match `kvspace.h`, frontend `codec_abi.c`, and `kvspace-c`: +// `langtype` is written verbatim into the attr trailer (NULL/empty -> empty attr). + +/// Hands ownership of `wire` to the caller (`free()`-able), or returns 1 if invalid/oversized. +fn emit(wire: Option>, out: *mut *mut u8, out_len: *mut u32) -> c_int { + if out.is_null() || out_len.is_null() { + return 1; + } + match wire { + Some(w) if u32::try_from(w.len()).is_ok() => alloc(w, out, out_len), + _ => 1, + } +} + +/// Scalar constructor: `ST_SHORT`, inline, `body = raw`. +fn new_scalar(raw: &[u8], langtype: *const c_char, out: *mut *mut u8, out_len: *mut u32) -> c_int { + let lt = unsafe { cstr(langtype) }; + emit( + crate::headlenpow::to_wire(ST_SHORT, REF_INLINE, raw, lt), + out, + out_len, + ) +} #[no_mangle] pub extern "C" fn kvspaceNewPtr( @@ -991,42 +1025,78 @@ pub extern "C" fn kvspaceNewPtr( out: *mut *mut u8, out_len: *mut u32, ) -> c_int { + if target_kindexpr.is_null() || target.is_null() { + return 1; + } let kindexpr = unsafe { cstr(target_kindexpr) }; // `any` 不是类型名（空 kindexpr = 不约束，见 spec [[文法与合法性]]）。 if kindexpr == "any" { return 1; } - let v = new_ptr(kindexpr, unsafe { cstr(target) }); - alloc(v.to_wire(), out, out_len) + // Write raw target path bytes as body; reject empty target (matches kvspaceXvalueHeadNewPtr). + let path = unsafe { CStr::from_ptr(target) }.to_bytes(); + if path.is_empty() { + return 1; + } + emit( + crate::headlenpow::to_wire(ST_SLACK, REF_PTR, path, kindexpr), + out, + out_len, + ) } #[no_mangle] pub extern "C" fn kvspaceNewChar( bytes: *const u8, len: u32, + langtype: *const c_char, out: *mut *mut u8, out_len: *mut u32, ) -> c_int { - let v = new_char_byte(unsafe { std::slice::from_raw_parts(bytes, len as usize) }); - alloc(v.to_wire(), out, out_len) + if len != 0 && bytes.is_null() { + return 1; + } + let body: &[u8] = if len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(bytes, len as usize) } + }; + let lt = unsafe { cstr(langtype) }; + emit( + crate::headlenpow::to_wire(ST_SLACK, REF_INLINE, body, lt), + out, + out_len, + ) } #[no_mangle] -pub extern "C" fn kvspaceNewBool(v: u8, out: *mut *mut u8, out_len: *mut u32) -> c_int { - let x = new_bool(&[v != 0]); - alloc(x.to_wire(), out, out_len) +pub extern "C" fn kvspaceNewBool( + v: u8, + langtype: *const c_char, + out: *mut *mut u8, + out_len: *mut u32, +) -> c_int { + new_scalar(&[u8::from(v != 0)], langtype, out, out_len) } #[no_mangle] -pub extern "C" fn kvspaceNewInt64(v: i64, out: *mut *mut u8, out_len: *mut u32) -> c_int { - let x = new_int64(&[v]); - alloc(x.to_wire(), out, out_len) +pub extern "C" fn kvspaceNewInt64( + v: i64, + langtype: *const c_char, + out: *mut *mut u8, + out_len: *mut u32, +) -> c_int { + new_scalar(&v.to_le_bytes(), langtype, out, out_len) } #[no_mangle] -pub extern "C" fn kvspaceNewFloat64(v: f64, out: *mut *mut u8, out_len: *mut u32) -> c_int { - let x = new_float64(&[v]); - alloc(x.to_wire(), out, out_len) +pub extern "C" fn kvspaceNewFloat64( + v: f64, + langtype: *const c_char, + out: *mut *mut u8, + out_len: *mut u32, +) -> c_int { + new_scalar(&v.to_bits().to_le_bytes(), langtype, out, out_len) } // ── slot view（shm-only）───────────────────────────────────────────── diff --git a/src/fs/kvspace.rs b/src/fs/kvspace.rs index b348bd0..4339081 100644 --- a/src/fs/kvspace.rs +++ b/src/fs/kvspace.rs @@ -304,7 +304,7 @@ impl FsKVSpace { continue; } let name = Self::unescape_name(&fname); - // RUNTIME_MEMBER_SEP（‥）前缀键是系统变量/langtype attr key，List 时隐藏。 + // Hide runtime system keys containing RUNTIME_MEMBER_SEP (‥) from List. if name.contains(RUNTIME_MEMBER_SEP) { continue; } @@ -595,7 +595,7 @@ impl KVSpace for FsKVSpace { if raw.is_empty() { return Err(format!("Cp: source not found: {}", src)); } - // attr key 与值同址：读解析后的物理 src 路径的类型。 + // Copy raw wire bytes verbatim (langtype lives in the value's attr trailer). let v = from_wire(&raw); self.set(&[KVPair { key: dst.to_string(), @@ -718,8 +718,8 @@ impl KVSpace for FsKVSpace { let resolved = self.resolve_path(path); self.ensure_dir(&resolved); if self.read_leaf(&resolved).is_none() { - // 固定头空目录 marker：storetype 0、ptr 0、空 body；类型 "lib" 在值 attr 区。 - let value = crate::headlenpow::to_wire(0, 0, &[], "lib") + // Directory marker: ST_SHORT, inline, empty body, langtype "lib" in attr. + let value = crate::headlenpow::to_wire(ST_SHORT, REF_INLINE, &[], "lib") .ok_or_else(|| format!("Mkindex: invalid directory {resolved}"))?; self.write_leaf(&resolved, &value); } @@ -741,8 +741,8 @@ impl KVSpace for FsKVSpace { return Err(format!("ExtIndex target missing: {ext_path}")); } let langtype = self.langtype_of(ext_path); - // 扩展定位：storetype 3（ST_EXT）+ ptr 2（PTR_EXT）+ 定位串 body + 类型 attr 区。 - let value = crate::headlenpow::to_wire(3, 2, ext_path.as_bytes(), &langtype) + // External locator: ST_EXT + REF_EXT + locator body + target langtype in attr. + let value = crate::headlenpow::to_wire(ST_EXT, REF_EXT, ext_path.as_bytes(), &langtype) .ok_or_else(|| format!("ExtIndex locator invalid: {ext_path}"))?; self.ensure_dir(&resolved); self.write_leaf(&resolved, &value); diff --git a/src/headlenpow.rs b/src/headlenpow.rs index a7b9c27..31e8256 100644 --- a/src/headlenpow.rs +++ b/src/headlenpow.rs @@ -74,47 +74,33 @@ pub fn attr_bytes(langtype: &str) -> Option> { /// 读 attr 总长（含自身 2 字节；空 attr = 2）。坏 wire → None。 #[inline] pub fn attr_len(data: &[u8], body_len: usize) -> Option { - let p = data.get(PREFIX.checked_add(body_len)?..PREFIX + body_len + 2)?; + let start = PREFIX.checked_add(body_len)?; + let p = data.get(start..start.checked_add(2)?)?; Some(u16::from_le_bytes([p[0], p[1]])) } /// 从 wire attr 区取 tag=1（langtype）记录 data；无/坏 attr → None。 +/// Scans only available bytes in `data` (mirrors C `kvspaceXvalueHeadLangtype`); truncated or +/// malformed records past `data.len()` return `None` without panicking. pub fn langtype_from_wire(data: &[u8], body_len: usize) -> Option<&[u8]> { let alen = usize::from(attr_len(data, body_len)?); - if alen < 2 { - return None; - } - let mut p = PREFIX + body_len + 2; - let end = PREFIX + body_len + alen; - while p + 3 <= end { - let tag = data[p]; - let rlen = usize::from(u16::from_le_bytes([data[p + 1], data[p + 2]])); - if p + 3 + rlen > end { - return None; + let start = PREFIX + body_len; // guaranteed <= data.len() - 2 by attr_len + let end = start.saturating_add(alen).min(data.len()); + let mut rest = data.get(start + 2..end)?; // None if alen < 2 + while let [tag, l0, l1, tail @ ..] = rest { + let rlen = usize::from(u16::from_le_bytes([*l0, *l1])); + let rec = tail.get(..rlen)?; + if *tag == ATTR_TAG_LANGTYPE { + return Some(rec); } - if tag == ATTR_TAG_LANGTYPE { - return Some(&data[p + 3..p + 3 + rlen]); - } - p += 3 + rlen; + rest = &tail[rlen..]; } None } pub fn to_wire(storetype: u8, ptr: u8, body: &[u8], langtype: &str) -> Option> { - if storetype > 3 || ptr > 2 { - return None; - } - let attr = attr_bytes(langtype)?; - let len = u64::try_from(body.len()).ok()?; - let total = PREFIX.checked_add(body.len())?.checked_add(attr.len())?; - let mut data = Vec::new(); - data.try_reserve_exact(total).ok()?; - data.resize(total, 0); - write_len(&mut data, len); - data[6] = storetype; - data[7] = ptr; + let mut data = reserve(storetype, ptr, body.len(), langtype)?; data[PREFIX..PREFIX + body.len()].copy_from_slice(body); - data[PREFIX + body.len()..].copy_from_slice(&attr); Some(data) } diff --git a/src/kvspace_common.rs b/src/kvspace_common.rs index d3116b2..0bc0aac 100644 --- a/src/kvspace_common.rs +++ b/src/kvspace_common.rs @@ -4,7 +4,7 @@ use std::time::Duration; use crate::kvspace::KVSpace; use crate::r#const::*; -use crate::xvalue::{body_bytes, is_none, plain, XValue}; +use crate::xvalue::{body_bytes, is_none, XValue}; /// JoinPath 拼接父子路径。 pub fn join_path(parent: &str, child: &str) -> String { @@ -192,100 +192,3 @@ pub fn equal_xvalue(a: &XValue, b: &XValue) -> bool { } body_bytes(a) == body_bytes(b) } - -// Display helpers. - -pub fn get_at(kv: &mut dyn KVSpace, dir: &str, name: &str) -> XValue { - kv.get(dir, &[name.to_string()], true).remove(0) -} - -pub fn read_prefix_ext(kv: &mut dyn KVSpace, prefix: &str) -> String { - if let XValue::Ext(e) = get_one(kv, prefix) { - return e.locator; - } - String::new() -} - -pub fn fprint_list(kv: &mut dyn KVSpace, prefix: &str, show_ext: bool, show_kind: bool) { - let children = kv.list(prefix, show_ext, true); - for c in children { - let mut v = get_at(kv, prefix, &c); - let child_dir = format!("{}{}", join_path(prefix, &c), DIR_INDEX_SUF); - let mut has_dir = !kv.list(&child_dir, false, true).is_empty(); - if !has_dir { - has_dir = !is_none(&get_at(kv, prefix, &format!("{}{}", c, DIR_INDEX_SUF))); - } - let mut key = c.clone(); - if has_dir { - key.push_str(DIR_INDEX_SUF); - v = XValue::None; - } - if is_none(&v) { - println!("{}", key); - } else if show_kind { - println!("{}\t{}\t{}", key, v.kind(), plain(&v)); - } else { - println!("{}\t{}", key, plain(&v)); - } - } - if !show_ext { - let ext = read_prefix_ext(kv, prefix); - if !ext.is_empty() { - println!("{}{}", EXT_INDEX_HEAD, ext); - for c in kv.list(&ext, false, true) { - println!(" {}", c); - } - } - } -} - -pub fn fprint_tree( - kv: &mut dyn KVSpace, - prefix: &str, - indent: &str, - show_ext: bool, - show_kind: bool, -) { - let mut children = kv.list(prefix, show_ext, true); - children.sort_by(|a, b| { - let (a_, b_) = (a.trim_end_matches('/'), b.trim_end_matches('/')); - if a_ == b_ { - a.ends_with('/').cmp(&b.ends_with('/')) - } else { - a_.cmp(b_) - } - }); - - let n = children.len(); - for (i, c) in children.iter().enumerate() { - let v = get_at(kv, prefix, c); - let base = c.trim_end_matches('/'); - let child_dir = format!("{}{}", join_path(prefix, base), DIR_INDEX_SUF); - let mut has_child = !kv.list(&child_dir, false, true).is_empty(); - if !has_child { - has_child = !is_none(&get_at(kv, prefix, &format!("{}{}", base, DIR_INDEX_SUF))); - } - - let last = i == n - 1; - let branch = if last { "└── " } else { "├── " }; - let next_indent = format!("{}{}", indent, if last { " " } else { "│ " }); - - if has_child && c.ends_with(DIR_INDEX_SUF) { - println!("{}{}{}", indent, branch, c); - fprint_tree(kv, &child_dir, &next_indent, show_ext, show_kind); - } else if is_none(&v) { - println!("{}{}{}", indent, branch, c); - } else if show_kind { - println!("{}{}{}\t{}\t{}", indent, branch, c, v.kind(), plain(&v)); - } else { - println!("{}{}{}\t{}", indent, branch, c, plain(&v)); - } - } - - if !show_ext { - let ext = read_prefix_ext(kv, prefix); - if !ext.is_empty() { - println!("{}└── {}{}", indent, EXT_INDEX_HEAD, ext); - } - } -} diff --git a/src/store.rs b/src/store.rs index 335197c..bad9872 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,7 +2,7 @@ pub(crate) fn is_subtree_key(prefix: &str, key: &str) -> bool { if prefix.is_empty() || prefix == "/" { return key.starts_with('/'); } - // '/'（层级）、'·'（成员）、'‥'（系统变量/langtype attr key）都是子树边界。 + // '/' (hierarchy), '·' (member), and '‥' (runtime system var) bound a subtree. key.strip_prefix(prefix).is_some_and(|rest| { rest.is_empty() || rest.starts_with('/') || rest.starts_with('·') || rest.starts_with('‥') }) diff --git a/src/xvalue.rs b/src/xvalue.rs index ba48e33..ba6c99f 100644 --- a/src/xvalue.rs +++ b/src/xvalue.rs @@ -434,27 +434,25 @@ fn array_to_header(kind: &str, array_len: i32) -> Vec { /// - dims 非空且非 map/struct → ST_LARGE（张量）； /// - 否则 ST_SHORT（None/标量/目录/map 头/空 rwfunc 目录值）。 pub fn to_wire_head(kind: &str, r#ref: i32, dims: &[i32], raw: &[u8]) -> Vec { - let storetype = if r#ref == 2 { - ST_EXT - } else if r#ref == 1 { - ST_SLACK - } else if r#ref != 0 { - return Vec::new(); - } else if matches!(kind, "rwir" | "def langtype") || (kind == "rwfunc" && !raw.is_empty()) { - ST_SLACK - } else if kind.starts_with("char/") { - ST_SLACK - } else if !dims.is_empty() && !kind.contains('·') && !kind.starts_with('/') { - ST_LARGE - } else { - ST_SHORT - }; - let ptr = match r#ref { - 1 => REF_PTR, - 2 => REF_EXT, - _ => REF_INLINE, + let (storetype, ptr) = match r#ref { + 0 => { + let slack = matches!(kind, "rwir" | "def langtype") + || (kind == "rwfunc" && !raw.is_empty()) + || kind.starts_with("char/"); + let st = if slack { + ST_SLACK + } else if !dims.is_empty() && !kind.contains(OBJ_SEP) && !kind.starts_with(PATH_SEP) { + ST_LARGE + } else { + ST_SHORT + }; + (st, REF_INLINE) + } + 1 => (ST_SLACK, REF_PTR), + 2 => (ST_EXT, REF_EXT), + _ => return Vec::new(), }; - let langtype = if kind == "None" { + let langtype = if kind == KIND_NONE { String::new() } else { kindexpr_of(kind, dims) @@ -467,35 +465,27 @@ pub fn from_wire_head(data: &[u8]) -> XValueHead { let Some(h) = crate::headlenpow::from_wire(data) else { return XValueHead::default(); }; - let Ok(headlen) = u16::try_from(crate::headlenpow::PREFIX) else { + let Ok(body_len) = i32::try_from(h.len) else { return XValueHead::default(); }; - let body_len = i32::try_from(h.len).unwrap_or(-1); - if body_len < 0 { - return XValueHead::default(); - } let langtype = crate::headlenpow::langtype_from_wire(data, h.len as usize) .map(|b| String::from_utf8_lossy(b).into_owned()) .unwrap_or_default(); - let dims = if h.storetype == 2 || (h.storetype == 1 && langtype.starts_with('[')) { + let has_dims = + h.storetype == ST_LARGE || (h.storetype == ST_SLACK && langtype.starts_with('[')); + let dims = if has_dims { parse_kindexpr(&langtype).0 } else { Vec::new() }; XValueHead { - headlen, - r#ref: if h.ptr == REF_PTR { - REF_PTR - } else if h.ptr == REF_EXT { - REF_EXT - } else { - REF_INLINE - }, + headlen: crate::headlenpow::PREFIX as u16, // fixed 8-byte header + r#ref: h.ptr, // validated <= 2 by headlenpow::from_wire; matches REF_* storetype: h.storetype, langtype, phys_dims: dims, body_len, - body_cap: h.len as u64, + body_cap: h.len, } } @@ -521,74 +511,6 @@ pub fn body_bytes(v: &XValue) -> Vec { } } -/// 返回 XValue 的完整 langtype 串（含 [dims] 形状段），用于写 x‥langtype attr key。 -/// None → 空串；Ptr/Ext → 目标 kindexpr；Map → 完整 map 类型；Opaque → 自定义 kind； -/// 数组 → `[d0,d1]kind`（无 dims 则裸 kind）。 -pub fn xvalue_langtype(v: &XValue) -> String { - let dims: &[i32] = match v { - XValue::None => return String::new(), - XValue::Ptr(p) => return p.target_kindexpr.clone(), - XValue::Ext(e) => return e.langtype.clone(), - XValue::Map(m) => return m.langtype.clone(), - XValue::Opaque(o) => return o.kind.clone(), - XValue::Bool(d) => &d.dims, - XValue::Int8(d) => &d.dims, - XValue::Int16(d) => &d.dims, - XValue::Int32(d) => &d.dims, - XValue::Int64(d) => &d.dims, - XValue::Uint8(d) => &d.dims, - XValue::Uint16(d) => &d.dims, - XValue::Uint32(d) => &d.dims, - XValue::Uint64(d) => &d.dims, - XValue::Float32(d) => &d.dims, - XValue::Float64(d) => &d.dims, - XValue::CharByte(d) => &d.dims, - XValue::CharAscii(d) => &d.dims, - XValue::Char32(d) => &d.dims, - }; - if dims.is_empty() { - v.kind().to_string() - } else { - format!( - "[{}]{}", - dims.iter() - .map(i32::to_string) - .collect::>() - .join(","), - v.kind() - ) - } -} - -/// ElemSize 返回 kind 的单元素字节数；≤0 表示非定长类型（非 byte 派生）。 -pub fn elem_size(kind: &str) -> i32 { - match kind { - KIND_INT8 | KIND_UINT8 | KIND_CHAR_UTF8 | KIND_CHAR_ASCII | KIND_BOOL => 1, - KIND_INT16 | KIND_UINT16 => 2, - KIND_INT32 | KIND_UINT32 | KIND_FLOAT32 | KIND_CHAR => 4, - KIND_INT64 | KIND_UINT64 | KIND_FLOAT64 | "time" | "duration" => 8, - _ => 0, - } -} - -/// Format 返回规范表示（对齐 Go 的 Format）。 -pub fn format(v: &XValue) -> String { - if is_none(v) { - KIND_NONE.to_string() - } else { - v.code_string() - } -} - -/// Plain 返回明文表示（对齐 Go 的 Plain）。 -pub fn plain(v: &XValue) -> String { - if is_none(v) { - KIND_NONE.to_string() - } else { - v.value_string() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/xvalue_byte.rs b/src/xvalue_byte.rs index 204dad4..fd927af 100644 --- a/src/xvalue_byte.rs +++ b/src/xvalue_byte.rs @@ -14,34 +14,6 @@ pub fn new_char_byte(v: &[u8]) -> XValue { dims: char_dims(v.len()), }) } -pub fn new_char32(v: &[u32]) -> XValue { - XValue::Char32(Arr { - data: v.to_vec(), - dims: char_dims(v.len()), - }) -} - -/// NewChar 根据 kind 从字符串构造字符值（char/utf32 默认，char/utf8/ascii 为字节）。 -pub fn new_char(kind: &str, s: &str) -> XValue { - match kind { - KIND_CHAR_UTF8 => XValue::CharByte(Arr { - data: s.as_bytes().to_vec(), - dims: char_dims(s.len()), - }), - KIND_CHAR_ASCII => XValue::CharAscii(Arr { - data: s.as_bytes().to_vec(), - dims: char_dims(s.len()), - }), - _ => { - let v: Vec = s.chars().map(|c| c as u32).collect(); - XValue::Char32(Arr { - data: v.clone(), - dims: char_dims(v.len()), - }) - } - } -} - pub fn char_byte_from_wire(body: &[u8], dims: &[i32]) -> Arr { Arr { data: body.to_vec(), @@ -74,8 +46,3 @@ pub fn char32_to_wire(data: &[u32], dims: &[i32]) -> Vec { let raw: Vec = data.iter().flat_map(|&v| v.to_le_bytes()).collect(); to_wire_head(KIND_CHAR, 0, dims, &raw) } - -/// IsCharKind 判断 kind 是否为字符家族（前缀 char/）。 -pub fn is_char_kind(kind: &str) -> bool { - kind.starts_with("char/") -} diff --git a/tests/metadata_fs.rs b/tests/metadata_fs.rs index c0f5de4..da7eb03 100644 --- a/tests/metadata_fs.rs +++ b/tests/metadata_fs.rs @@ -33,7 +33,7 @@ fn value_and_directory_keys_coexist() { kv.mkindex("/node/", 0).unwrap(); let directory = kv.get_raw("/node/"); let dh = kvspace_durable::headlenpow::from_wire(&directory).unwrap(); - assert_eq!(dh.storetype, 0); /* 目录 = storetype 0 空 body；langtype 在 attr key */ + assert_eq!(dh.storetype, 0); /* directory marker: storetype 0, empty body, langtype in attr */ assert_eq!(dh.ptr, 0); assert_eq!(kv.get_raw("/node"), value); for name in ["__dir__", "__self__", "~abc"] { diff --git a/tests/prefix_list.rs b/tests/prefix_list.rs index 3749ae8..fda25c1 100644 --- a/tests/prefix_list.rs +++ b/tests/prefix_list.rs @@ -135,7 +135,6 @@ fn new_wire_directories_and_extensions_use_physical_keys() { .0 .borrow() .iter() - .filter(|(key, _)| !key.ends_with("‥langtype")) .all(|(_, raw)| kvspace_durable::headlenpow::from_wire(raw).is_some())); assert!(inspect.get("/m·").is_none()); } From c0a2a9a3d2e1d511388fa63c7d8cb3a9408fc8e0 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Fri, 30 Oct 2026 17:30:20 +0000 Subject: [PATCH 02/12] kvspace-durable: reconnect Redis on I/O errors, catch backend panics, atomic fs writes, and SCAN MATCH - redis: drop TCP connection on I/O error or timeout so delayed replies cannot desynchronize subsequent commands - ffi: wrap lazy flush, WriteInPlace readback, and Watch polling in catch_unwind to return 1 instead of aborting across C ABI - fs: overwrite body in place when layout matches, otherwise write a temp file and rename atomically - ffi: add per-handle child list snapshot for ListLen/ListAt - redis: pass escaped MATCH prefix to server-side SCAN --- src/ffi.rs | 112 +++++++++++++++++++++----- src/fs/kvspace.rs | 62 ++++++++++++++- src/redis/store.rs | 192 ++++++++++++++++++++++++++++----------------- 3 files changed, 275 insertions(+), 91 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 34d2176..0c28244 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -53,24 +53,74 @@ pub struct Handle { retired: Vec>, write_buf: Vec, pending_key: Option, + /// Cached child list for `ListLen` followed by sequential `ListAt(i)` traversal. + list_snapshot: Option, +} + +struct ListSnapshot { + prefix: String, + expand_ext: bool, + resolve: bool, + names: Vec, } impl Handle { /// 落盘上一笔惰性写（调用方已把 body 填进 write_buf），并失效该 key 的读缓存。 + /// Catches backend panics (Redis disconnect/timeout, fs I/O error) and converts them into + /// `Err` so they never unwind across `extern "C"`. fn flush(&mut self) -> Result<(), String> { if let Some(key) = self.pending_key.take() { let xv = std::mem::take(&mut self.write_buf); - let val = from_wire(&xv); - self.kv.set(&[KVPair { - key: key.clone(), - val, - raw: Some(xv), - }])?; + self.list_snapshot = None; + let kv = &mut self.kv; + let k = key.clone(); + let r = catch_panic(move || { + let val = from_wire(&xv); + kv.set(&[KVPair { + key: k, + val, + raw: Some(xv), + }]) + }); self.read_cache.remove(&key); + r?; } Ok(()) } + /// Returns child names under `prefix`, reusing the snapshot when valid. + fn list_names( + &mut self, + prefix: &str, + expand_ext: bool, + resolve: bool, + fresh: bool, + ) -> Result<&[String], String> { + let reuse = !fresh + && self.list_snapshot.as_ref().is_some_and(|s| { + s.prefix == prefix && s.expand_ext == expand_ext && s.resolve == resolve + }); + if !reuse { + self.list_snapshot = None; + let kv = &mut self.kv; + let names = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + kv.list(prefix, expand_ext, resolve) + })) + .map_err(panic_msg)?; + self.list_snapshot = Some(ListSnapshot { + prefix: prefix.to_string(), + expand_ext, + resolve, + names, + }); + } + Ok(&self + .list_snapshot + .as_ref() + .expect("snapshot just set") + .names) + } + fn cache_invalidate(&mut self, key: &str) { self.read_cache.remove(key); self.volatile_cache.remove(key); @@ -182,8 +232,10 @@ fn volatile_get_part( } /// flush 上一笔惰性写后返回后端引用。任一步失败 → Err（调用方转 err/err 返回 1）。 +/// Used by mutating structure ops (`Del`/`DelTree`/`Cp*`/`Mkindex*`/`RmindexExt`/`Clear`); clears `list_cache`. unsafe fn kv_flush<'a>(h: *mut Handle) -> Result<&'a mut dyn KVSpace, String> { let hd = h.as_mut().ok_or("kvspace: null handle")?; + hd.list_snapshot = None; hd.flush()?; Ok(&mut *hd.kv) } @@ -326,6 +378,7 @@ pub extern "C" fn kvspaceConnect(dsn: *const c_char) -> *mut Handle { retired: Vec::new(), write_buf: Vec::new(), pending_key: None, + list_snapshot: None, })), Err(_) => std::ptr::null_mut(), } @@ -433,6 +486,7 @@ pub extern "C" fn kvspaceSetValue( return 1; } let raw = raw.to_vec(); + hd.list_snapshot = None; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let val = from_wire(&raw); hd.kv.set(&[KVPair { @@ -541,6 +595,7 @@ pub extern "C" fn kvspaceSetPart( } let key = unsafe { cstr(key) }.to_string(); let data = unsafe { std::slice::from_raw_parts(buf, buf_len as usize) }; + hd.list_snapshot = None; let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { hd.kv.set_part(&key, offset, data) })) @@ -606,7 +661,14 @@ pub extern "C" fn kvspaceWriteInPlace( return 1; } let key = unsafe { cstr(key) }.to_string(); - let existing = hd.kv.get_raw(&key); + let existing = + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(&key))) { + Ok(v) => v, + Err(p) => { + write_err(err, err_cap, &panic_msg(p)); + return 1; + } + }; if existing.is_empty() { write_err(err, err_cap, "kvspace: write-in-place on missing key"); return 1; @@ -690,11 +752,10 @@ pub extern "C" fn kvspaceListLen( unsafe { *out_count = 0 }; return 1; } - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - hd.kv.list_len(&prefix, expand_ext != 0, resolve != 0) - })) { - Ok(n) => { - unsafe { *out_count = n }; + /* Always re-list so callers polling child count observe external additions; snapshot serves ListAt. */ + match hd.list_names(&prefix, expand_ext != 0, resolve != 0, true) { + Ok(names) => { + unsafe { *out_count = names.len() as i32 }; 0 } Err(_) => { @@ -726,14 +787,15 @@ pub extern "C" fn kvspaceListAt( *out_len = 0; } let prefix = unsafe { cstr(prefix) }.to_string(); - if hd.flush().is_err() || hd.kv.validate_dir(&prefix).is_err() { + if idx < 0 || hd.flush().is_err() || hd.kv.validate_dir(&prefix).is_err() { return 1; } - let name = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - hd.kv.list_at(&prefix, idx, expand_ext != 0, resolve != 0) - })) { - Ok(Some(n)) => n, - Ok(None) => return 1, + /* Reuse snapshot when parameters match without intervening writes. */ + let name = match hd.list_names(&prefix, expand_ext != 0, resolve != 0, false) { + Ok(names) => match names.get(idx as usize) { + Some(n) => n.clone(), + None => return 1, + }, Err(_) => return 1, }; let name = name.as_bytes(); @@ -961,7 +1023,19 @@ pub extern "C" fn kvspaceWatch( let target_v = from_wire(unsafe { std::slice::from_raw_parts(target, target_len as usize) }); - let v = hd.kv.watch(&key, &target_v, Duration::from_nanos(tick_ns)); + let kv = &mut hd.kv; + let v = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + kv.watch(&key, &target_v, Duration::from_nanos(tick_ns)) + })) { + Ok(v) => v, + Err(_) => { + unsafe { + *out = std::ptr::null_mut(); + *out_len = 0; + } + return 1; + } + }; if let Some(old) = hd.read_cache.remove(&key) { hd.retired.push(old); /* 旧视图保活到 Close（watch 是冷路径） */ } diff --git a/src/fs/kvspace.rs b/src/fs/kvspace.rs index 4339081..ab5df7f 100644 --- a/src/fs/kvspace.rs +++ b/src/fs/kvspace.rs @@ -17,6 +17,55 @@ use crate::r#const::*; use crate::xvalue::*; const SELF_MARKER: &str = "__self__"; const DIR_MARKER: &str = "__dir__"; +/// Prefix for atomic-write temp files (`~~` cannot collide with `~` + hex escaped key names) +/// and is skipped during directory listing and copies. +const TMP_PREFIX: &str = "~~tmp."; +static TMP_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Replaces a full value without exposing a missing or truncated file to concurrent readers: +/// overwrites the body in place when length, header, and attr match; otherwise writes a temp +/// file in the same directory and renames it over `path`. +fn write_atomic(path: &Path, val: &[u8]) -> std::io::Result<()> { + if overwrite_body_in_place(path, val)? { + return Ok(()); + } + let dir = path.parent().unwrap_or_else(|| Path::new(".")); + let seq = TMP_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let tmp = dir.join(format!("{TMP_PREFIX}{}.{seq}", std::process::id())); + fs::write(&tmp, val)?; + fs::rename(&tmp, path).inspect_err(|_| { + let _ = fs::remove_file(&tmp); + }) +} + +/// Overwrites only the body if `path` already has the same size, header, and attr bytes. +fn overwrite_body_in_place(path: &Path, val: &[u8]) -> std::io::Result { + let Some(head) = crate::headlenpow::from_wire(val) else { + return Ok(false); + }; + let attr_off = crate::headlenpow::PREFIX + head.body.len(); + let f = match fs::OpenOptions::new().read(true).write(true).open(path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(e), + }; + if f.metadata()?.len() != val.len() as u64 { + return Ok(false); + } + let mut old = vec![0u8; crate::headlenpow::PREFIX + (val.len() - attr_off)]; + let (old_head, old_attr) = old.split_at_mut(crate::headlenpow::PREFIX); + f.read_exact_at(old_head, 0)?; + f.read_exact_at(old_attr, attr_off as u64)?; + if old_head != &val[..crate::headlenpow::PREFIX] || old_attr != &val[attr_off..] { + return Ok(false); + } + f.write_all_at(head.body, crate::headlenpow::PREFIX as u64)?; + Ok(true) +} + +fn is_tmp_name(name: &std::ffi::OsStr) -> bool { + name.to_str().is_some_and(|n| n.starts_with(TMP_PREFIX)) +} pub struct FsKVSpace { root: PathBuf, @@ -161,13 +210,13 @@ impl FsKVSpace { } let p = self.fs_path(key); if p.is_dir() { - fs::write(p.join(Self::leaf_marker(key)), val) + write_atomic(&p.join(Self::leaf_marker(key)), val) .unwrap_or_else(|e| panic!("kvspace-fs: set {}: {}", key, e)); } else { if let Some(parent) = p.parent() { let _ = fs::create_dir_all(parent); } - fs::write(p, val).unwrap_or_else(|e| panic!("kvspace-fs: set {}: {}", key, e)); + write_atomic(&p, val).unwrap_or_else(|e| panic!("kvspace-fs: set {}: {}", key, e)); } } @@ -296,6 +345,9 @@ impl FsKVSpace { let mut children = Vec::new(); if let Ok(entries) = fs::read_dir(&p) { for e in entries.flatten() { + if is_tmp_name(&e.file_name()) { + continue; + } let fname = e.file_name().to_string_lossy().into_owned(); if fname == SELF_MARKER || fname == DIR_MARKER @@ -329,6 +381,9 @@ impl FsKVSpace { fs::create_dir_all(dst)?; for e in fs::read_dir(src)? { let e = e?; + if is_tmp_name(&e.file_name()) { + continue; + } Self::copy_recursive(&e.path(), &dst.join(e.file_name()))?; } } else { @@ -686,6 +741,9 @@ impl KVSpace for FsKVSpace { let _ = fs::create_dir_all(&dmem); if let Ok(rd) = fs::read_dir(&smem) { for e in rd.flatten() { + if is_tmp_name(&e.file_name()) { + continue; + } let sp = e.path(); let dp = dmem.join(e.file_name()); if sp.is_file() { diff --git a/src/redis/store.rs b/src/redis/store.rs index 7dc1bf9..bdebf21 100644 --- a/src/redis/store.rs +++ b/src/redis/store.rs @@ -1,13 +1,18 @@ // redis/store.rs — Redis 存储原语（最小 RESP 客户端，仅 GET/SET/DEL/SCAN/FLUSHDB）。 use std::cell::RefCell; -use std::io::{BufRead, BufReader, Read, Write}; +use std::io::{self, BufRead, BufReader, Read, Write}; use std::net::TcpStream; use crate::store::{is_subtree_key, KVStore}; +type Conn = BufReader; + pub struct RedisStore { - stream: RefCell>, + addr: String, + /// Set to `None` on I/O error or timeout so the next command reconnects instead of + /// reading a delayed response left in the stream. + stream: RefCell>, } pub fn connect(addr: &str) -> RedisStore { @@ -21,91 +26,129 @@ pub fn connect(addr: &str) -> RedisStore { impl RedisStore { pub fn new(addr: &str) -> Self { - let stream = TcpStream::connect(addr) - .unwrap_or_else(|e| panic!("kvspace-redis: connect {}: {}", addr, e)); - stream - .set_read_timeout(Some(std::time::Duration::from_secs(3))) - .ok(); - stream - .set_write_timeout(Some(std::time::Duration::from_secs(3))) - .ok(); - stream.set_nodelay(true).ok(); // 请求/应答 ping-pong，禁 Nagle 避免与延迟 ACK 叠加卡顿 + let conn = + Self::open(addr).unwrap_or_else(|e| panic!("kvspace-redis: connect {}: {}", addr, e)); RedisStore { - stream: RefCell::new(BufReader::new(stream)), // 缓冲读：RESP 逐字节 read_exact 曾致 syscall 风暴 + addr: addr.to_string(), + stream: RefCell::new(Some(conn)), } } + fn open(addr: &str) -> io::Result { + let stream = TcpStream::connect(addr)?; + stream.set_read_timeout(Some(std::time::Duration::from_secs(3)))?; + stream.set_write_timeout(Some(std::time::Duration::from_secs(3)))?; + stream.set_nodelay(true)?; // disable Nagle for request/reply round-trips + Ok(BufReader::new(stream)) // buffer RESP reads to avoid per-byte syscalls + } + fn cmd(&self, args: &[&[u8]]) -> Resp { - { - let mut s = self.stream.borrow_mut(); - let mut out = format!("*{}\r\n", args.len()).into_bytes(); - for a in args { - out.extend_from_slice(format!("${}\r\n", a.len()).as_bytes()); - out.extend_from_slice(a); - out.extend_from_slice(b"\r\n"); + let response = { + let mut slot = self.stream.borrow_mut(); + let r = Self::round_trip(&self.addr, &mut slot, args); + if r.is_err() { + *slot = None; // drop desynchronized stream; reconnect on next command } - s.get_mut() - .write_all(&out) - .unwrap_or_else(|e| panic!("kvspace-redis: write: {}", e)); + r + }; + match response { + // Server error reply is a complete frame; stream remains aligned. + Ok(Resp::Error(message)) => panic!("kvspace-redis: {message}"), + Ok(r) => r, + Err(e) => panic!("kvspace-redis: {}: {e}", self.addr), } - let response = self.read_resp(); - if let Resp::Error(message) = &response { - panic!("kvspace-redis: {message}"); - } - response } - fn read_line(&self) -> Vec { - let mut s = self.stream.borrow_mut(); - let mut line = Vec::new(); - s.read_until(b'\n', &mut line) - .unwrap_or_else(|e| panic!("kvspace-redis: read: {}", e)); - if line.ends_with(b"\n") { - line.pop(); + fn round_trip(addr: &str, slot: &mut Option, args: &[&[u8]]) -> io::Result { + if slot.is_none() { + *slot = Some(Self::open(addr)?); } - if line.ends_with(b"\r") { - line.pop(); + let conn = slot.as_mut().expect("connection just opened"); + let mut out = format!("*{}\r\n", args.len()).into_bytes(); + for a in args { + out.extend_from_slice(format!("${}\r\n", a.len()).as_bytes()); + out.extend_from_slice(a); + out.extend_from_slice(b"\r\n"); } - line + conn.get_mut().write_all(&out)?; + read_resp(conn) + } +} + +fn read_line(conn: &mut Conn) -> io::Result> { + let mut line = Vec::new(); + if conn.read_until(b'\n', &mut line)? == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "connection closed", + )); + } + if line.ends_with(b"\n") { + line.pop(); } + if line.ends_with(b"\r") { + line.pop(); + } + Ok(line) +} - fn read_resp(&self) -> Resp { - let line = self.read_line(); - if line.is_empty() { - return Resp::Error("empty response".to_string()); +fn read_resp(conn: &mut Conn) -> io::Result { + let line = read_line(conn)?; + let Some((&kind, rest)) = line.split_first() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "empty RESP line", + )); + }; + let text = || String::from_utf8_lossy(rest).into_owned(); + Ok(match kind { + b'+' => Resp::Simple(text()), + b'-' => Resp::Error(text()), + b':' => Resp::Integer(text().parse().unwrap_or(0)), + b'$' => { + let len: i64 = text().parse().unwrap_or(-1); + if len < 0 { + return Ok(Resp::Bulk(None)); + } + let mut data = vec![0u8; len as usize]; + conn.read_exact(&mut data)?; + let mut crlf = [0u8; 2]; + conn.read_exact(&mut crlf)?; + Resp::Bulk(Some(data)) } - match line[0] { - b'+' => Resp::Simple(String::from_utf8_lossy(&line[1..]).into_owned()), - b'-' => Resp::Error(String::from_utf8_lossy(&line[1..]).into_owned()), - b':' => Resp::Integer(String::from_utf8_lossy(&line[1..]).parse().unwrap_or(0)), - b'$' => { - let len: i64 = String::from_utf8_lossy(&line[1..]).parse().unwrap_or(-1); - if len < 0 { - return Resp::Bulk(None); - } - let mut s = self.stream.borrow_mut(); - let mut data = vec![0u8; len as usize]; - s.read_exact(&mut data) - .unwrap_or_else(|e| panic!("kvspace-redis: read bulk: {}", e)); - let mut crlf = [0u8; 2]; - s.read_exact(&mut crlf) - .unwrap_or_else(|e| panic!("kvspace-redis: read crlf: {}", e)); - Resp::Bulk(Some(data)) + b'*' => { + let n: i64 = text().parse().unwrap_or(-1); + if n < 0 { + return Ok(Resp::Array(Vec::new())); } - b'*' => { - let n: i64 = String::from_utf8_lossy(&line[1..]).parse().unwrap_or(-1); - if n < 0 { - return Resp::Array(Vec::new()); - } - let mut arr = Vec::with_capacity(n as usize); - for _ in 0..n { - arr.push(self.read_resp()); - } - Resp::Array(arr) + let mut arr = Vec::with_capacity(n as usize); + for _ in 0..n { + arr.push(read_resp(conn)?); } - _ => Resp::Error(format!("unexpected RESP type: {}", line[0] as char)), + Resp::Array(arr) + } + // Unrecognized RESP type byte indicates stream desynchronization. + other => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected RESP type: {}", other as char), + )) + } + }) +} + +/// Builds a SCAN MATCH glob (`* ? [ ] \` escaped, plus trailing `*`; empty prefix -> `/*`). +fn scan_pattern(prefix: &str) -> Vec { + let base = if prefix.is_empty() { "/" } else { prefix }; + let mut pat = Vec::with_capacity(base.len() + 1); + for &b in base.as_bytes() { + if matches!(b, b'*' | b'?' | b'[' | b']' | b'\\') { + pat.push(b'\\'); } + pat.push(b); } + pat.push(b'*'); + pat } enum Resp { @@ -184,11 +227,20 @@ impl KVStore for RedisStore { } fn scan_keys(&self, prefix: &str) -> Vec { + // Filter by prefix server-side via MATCH; check subtree boundaries locally via is_subtree_key. + let pattern = scan_pattern(prefix); let mut keys = Vec::new(); let mut cursor: i64 = 0; loop { let cur = cursor.to_string(); - match self.cmd(&[b"SCAN", cur.as_bytes(), b"COUNT", b"1000"]) { + match self.cmd(&[ + b"SCAN", + cur.as_bytes(), + b"MATCH", + &pattern, + b"COUNT", + b"1000", + ]) { Resp::Array(parts) if parts.len() == 2 => { cursor = match &parts[0] { Resp::Bulk(Some(bytes)) => { From f88ffd43a8874c20891be38d10c5883576063f40 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 05:42:11 +0000 Subject: [PATCH 03/12] ffi: cache per-traversal child list so ListLen + sequential ListAt is O(N) Reuse the last list() snapshot in kvspaceListAt while (prefix, expand_ext, resolve) match, idx strictly increases, and no writes occur on the handle. Re-list on ListLen, any handle mutation, or repeated/non-increasing idx so polling callers observe external updates. --- src/ffi.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 7dedc6c..518c476 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -56,12 +56,25 @@ pub struct Handle { retired: Vec>, write_buf: Vec, pending_key: Option, + /// Cached child list for `ListLen` + sequential `ListAt` traversal. + list_cache: Option, +} + +/// Per-traversal child list snapshot reused while `(prefix, expand_ext, resolve)` match and `idx` increases. +struct ListCache { + prefix: String, + expand_ext: bool, + resolve: bool, + names: Vec, + /// Last yielded index. + cursor: i32, } impl Handle { /// 落盘上一笔惰性写（调用方已把 body 填进 write_buf），并失效该 key 的读缓存。 fn flush(&mut self) -> Result<(), String> { if let Some(key) = self.pending_key.take() { + self.list_cache = None; let xv = std::mem::take(&mut self.write_buf); let val = from_wire(&xv); self.kv.set(&[KVPair { @@ -170,9 +183,11 @@ fn volatile_get_part( } /// flush 上一笔惰性写后返回后端引用。任一步失败 → Err（调用方转 err/err 返回 1）。 +/// Flushes pending lazy writes and clears `list_cache` before mutating structure ops. unsafe fn kv_flush<'a>(h: *mut Handle) -> Result<&'a mut dyn KVSpace, String> { let hd = h.as_mut().ok_or("kvspace: null handle")?; hd.flush()?; + hd.list_cache = None; Ok(&mut *hd.kv) } @@ -314,6 +329,7 @@ pub extern "C" fn kvspaceConnect(dsn: *const c_char) -> *mut Handle { retired: Vec::new(), write_buf: Vec::new(), pending_key: None, + list_cache: None, })), Err(_) => std::ptr::null_mut(), } @@ -421,6 +437,7 @@ pub extern "C" fn kvspaceSetValue( return 1; } let raw = raw.to_vec(); + hd.list_cache = None; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let val = from_wire(&raw); hd.kv.set(&[KVPair { @@ -529,6 +546,7 @@ pub extern "C" fn kvspaceSetPart( } let key = unsafe { cstr(key) }.to_string(); let data = unsafe { std::slice::from_raw_parts(buf, buf_len as usize) }; + hd.list_cache = None; let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { hd.kv.set_part(&key, offset, data) })) @@ -674,6 +692,8 @@ pub extern "C" fn kvspaceListLen( None => return 1, }; let prefix = unsafe { cstr(prefix) }.to_string(); + // Always re-read backend on ListLen; the first ListAt builds the new traversal snapshot. + hd.list_cache = None; if hd.flush().is_err() || hd.kv.validate_dir(&prefix).is_err() { unsafe { *out_count = 0 }; return 1; @@ -717,12 +737,34 @@ pub extern "C" fn kvspaceListAt( if hd.flush().is_err() || hd.kv.validate_dir(&prefix).is_err() { return 1; } - let name = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - hd.kv.list_at(&prefix, idx, expand_ext != 0, resolve != 0) - })) { - Ok(Some(n)) => n, - Ok(None) => return 1, - Err(_) => return 1, + if idx < 0 { + return 1; + } + let (ex, res) = (expand_ext != 0, resolve != 0); + // Reuse snapshot when `(prefix, ex, res)` match and `idx` strictly increases without writes; + // repeated or non-increasing `idx` re-lists so polling the same `idx` stays fresh. + let hit = matches!(&hd.list_cache, Some(c) if c.prefix == prefix + && c.expand_ext == ex && c.resolve == res && idx > c.cursor); + if !hit { + hd.list_cache = None; + let names = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + hd.kv.list(&prefix, ex, res) + })) { + Ok(n) => n, + Err(_) => return 1, + }; + hd.list_cache = Some(ListCache { + prefix, + expand_ext: ex, + resolve: res, + names, + cursor: idx, + }); + } + let c = hd.list_cache.as_mut().unwrap(); + c.cursor = idx; + let Some(name) = c.names.get(idx as usize) else { + return 1; }; let name = name.as_bytes(); unsafe { From a4bc3a4a8bf51f54f938bcb70be5822ee24e7c95 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 14:28:40 +0000 Subject: [PATCH 04/12] tests: add C ABI regression test for ListLen/ListAt snapshot visibility Verify sequential ListAt reuse, immediate invalidation on same-handle mutations (SetValue, Del, DelTree, lazy WriteNewPlace), and cross-handle visibility on repeated idx or ListLen. --- tests/list_cache.rs | 174 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tests/list_cache.rs diff --git a/tests/list_cache.rs b/tests/list_cache.rs new file mode 100644 index 0000000..6a17bfe --- /dev/null +++ b/tests/list_cache.rs @@ -0,0 +1,174 @@ +// list_cache.rs - Visibility of per-handle ListLen/ListAt snapshots (C ABI, fs backend). +// +// Compares every step against a freshly opened handle without a cached snapshot: +// 1. ListLen + sequential ListAt(0..n) matches a fresh listing; +// 2. Same-handle mutations (SetValue / Del / DelTree / lazy WriteNewPlace) take effect immediately; +// 3. Cross-handle writes become visible on repeated `idx` or the next `ListLen`. +use kvspace_durable::ffi::*; +use std::ffi::CString; + +fn xvalue_i64(n: i64) -> Vec { + let mut v = vec![0u8; 16]; + v[0..6].copy_from_slice(&8u64.to_le_bytes()[..6]); + v[8..16].copy_from_slice(&n.to_le_bytes()); + v.extend_from_slice(&[2, 0]); // empty attr + v +} + +unsafe fn set(h: *mut Handle, key: &str, n: i64) { + let k = CString::new(key).unwrap(); + let v = xvalue_i64(n); + let rc = kvspaceSetValue( + h, + k.as_ptr(), + v.as_ptr(), + v.len() as u32, + std::ptr::null_mut(), + 0, + ); + assert_eq!(rc, 0, "set {key}"); +} + +unsafe fn del(h: *mut Handle, key: &str) { + let k = CString::new(key).unwrap(); + let ks = [k.as_ptr()]; + assert_eq!( + kvspaceDel(h, ks.as_ptr(), 1, std::ptr::null_mut(), 0), + 0, + "del {key}" + ); +} + +unsafe fn list_len(h: *mut Handle, dir: &str) -> i32 { + let d = CString::new(dir).unwrap(); + let mut n = -1; + assert_eq!(kvspaceListLen(h, d.as_ptr(), 0, 0, &mut n), 0); + n +} + +unsafe fn list_at(h: *mut Handle, dir: &str, idx: i32) -> Option { + let d = CString::new(dir).unwrap(); + let mut buf = [0u8; 256]; + let mut ol = 0u32; + if kvspaceListAt( + h, + d.as_ptr(), + 0, + 0, + idx, + buf.as_mut_ptr(), + buf.len() as u32, + &mut ol, + ) != 0 + { + return None; + } + Some(String::from_utf8(buf[..ol as usize].to_vec()).unwrap()) +} + +unsafe fn traverse(h: *mut Handle, dir: &str) -> Vec { + let n = list_len(h, dir); + (0..n) + .map(|i| list_at(h, dir, i).expect("ListAt in range")) + .collect() +} + +/// Uncached reference listing via a fresh handle. +unsafe fn fresh(dsn: &CString, dir: &str) -> Vec { + let h = kvspaceConnect(dsn.as_ptr()); + assert!(!h.is_null()); + let v = traverse(h, dir); + kvspaceClose(h); + v +} + +#[test] +fn list_cache_visibility_fs() { + let dir = std::env::temp_dir().join(format!("kvspace-list-cache-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let dsn = CString::new(format!("fs://{}", dir.display())).unwrap(); + unsafe { + let h = kvspaceConnect(dsn.as_ptr()); + let h2 = kvspaceConnect(dsn.as_ptr()); + assert!(!h.is_null() && !h2.is_null()); + let d = "/d/"; + let md = CString::new(d).unwrap(); + assert_eq!( + kvspaceMkindex(h, md.as_ptr(), 0, std::ptr::null_mut(), 0), + 0 + ); + for k in ["a", "b", "c"] { + set(h, &format!("/d/{k}"), 1); + } + // 1. Sequential traversal matches a fresh listing. + assert_eq!(traverse(h, d), fresh(&dsn, d)); + assert_eq!(traverse(h, d).len(), 3); + + // 2. Same-handle write mid-traversal is visible on the next index. + assert_eq!(list_len(h, d), 3); + let first = list_at(h, d, 0).unwrap(); + set(h, "/d/0", 1); + let want = fresh(&dsn, d); + assert_eq!(want.len(), 4); + assert_eq!( + list_at(h, d, 1), + Some(want[1].clone()), + "own SetValue mid-traversal" + ); + assert_ne!(first, ""); + del(h, "/d/0"); + assert_eq!(list_at(h, d, 3), None, "own Del mid-traversal"); + assert_eq!(traverse(h, d), fresh(&dsn, d)); + + // Lazy WriteNewPlace flushes on the next call and invalidates the snapshot. + assert_eq!(list_len(h, d), 3); + let k = CString::new("/d/e").unwrap(); + let lt = CString::new("int64").unwrap(); + let mut body: *mut u8 = std::ptr::null_mut(); + assert_eq!( + kvspaceWriteNewPlace( + h, + k.as_ptr(), + 0, + 0, + 8, + 8, + lt.as_ptr(), + &mut body, + std::ptr::null_mut(), + 0 + ), + 0 + ); + std::ptr::write_bytes(body, 0, 8); + assert_eq!( + list_at(h, d, 3), + Some("e".to_string()), + "lazy write visible" + ); + + // DelTree invalidates the snapshot. + let sub = CString::new("/d/e").unwrap(); + assert_eq!(list_len(h, d), 4); + assert_eq!(kvspaceDelTree(h, sub.as_ptr(), std::ptr::null_mut(), 0), 0); + assert_eq!(list_at(h, d, 3), None, "DelTree mid-traversal"); + + // 3. Cross-handle write becomes visible when repeating the same index. + let before = traverse(h, d); + assert_eq!(list_at(h, d, 0), Some(before[0].clone())); + set(h2, "/d/0", 2); + let want = fresh(&dsn, d); + assert_eq!( + list_at(h, d, 0), + Some(want[0].clone()), + "repeated idx re-lists" + ); + del(h2, "/d/0"); + assert_eq!(list_len(h, d), 3, "ListLen re-lists"); + assert_eq!(traverse(h, d), fresh(&dsn, d)); + + kvspaceClose(h2); + kvspaceClose(h); + } + let _ = std::fs::remove_dir_all(&dir); +} From 072d25cb70db4bb0b57a2ccb4920d078dace14af Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 14:36:04 +0000 Subject: [PATCH 05/12] kvspace-durable: align DelTree deletion scope and link handling across backends - unify subtree deletion around store::is_subtree_key(base, key) via subtree_base - fs: remove sibling runtime-attribute entries in the parent directory, preserve the root directory itself on DelTree("/"), and resolve only parent links so DelTree on a link deletes only the link - backend (redis/s3): resolve only parent components before checking whether the target key is a link - tests: add tests/deltree.rs covering fs and Redis --- src/backend.rs | 41 +++---- src/fs/kvspace.rs | 75 +++++++++--- src/kvspace_common.rs | 9 ++ tests/deltree.rs | 277 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 360 insertions(+), 42 deletions(-) create mode 100644 tests/deltree.rs diff --git a/src/backend.rs b/src/backend.rs index aa714fa..6414df1 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -6,7 +6,7 @@ use std::time::Duration; use crate::kvspace::{KVPair, KVSpace}; use crate::kvspace_common::{ is_descendant, is_frame_operand_key, join_path, sep_path, split_index, strip_dir_suf, - validate_ptr, watch_value, + subtree_base, validate_ptr, watch_value, }; use crate::r#const::*; use crate::store::KVStore; @@ -436,36 +436,33 @@ impl KVSpace for Backend { if crate::r#const::is_reserved(prefix) { return Err(format!("reserved metadata key: {prefix}")); } - let mut link_key = prefix; - if Self::is_dir(link_key) && link_key != PATH_SEP { - link_key = strip_dir_suf(prefix); - } - if let Some(data) = self.store.get(link_key) { - let head = from_wire_head(&data); - if head.is_ptr() { - return self.del(&[prefix.to_string()]); - } - } - - let resolved = self.resolve_path(prefix); + // Resolve links only on parent components (POSIX rm semantics); if the final + // component is itself a link, delete the link rather than its target subtree. + let resolved = self.resolve_parent(prefix); if crate::r#const::is_reserved(&resolved) { return Err(format!("reserved metadata key: {resolved}")); } - let mut scan = resolved.clone(); - if Self::is_dir(&scan) && scan != PATH_SEP { - scan.pop(); + let base = subtree_base(&resolved); + if !base.is_empty() { + if let Some(data) = self.store.get(base) { + if from_wire_head(&data).is_ptr() { + return self.del(std::slice::from_ref(&resolved)); + } + } } + + // Delete base and all descendants bounded by '/', '·', or '‥' (via is_subtree_key). + let scan = if base.is_empty() { PATH_SEP } else { base }; let keys: Vec<_> = self .store - .scan_keys(&scan) + .scan_keys(scan) .into_iter() .filter(|key| !crate::r#const::is_reserved(key)) .collect(); - - let mut del_keys: Vec<&str> = Vec::with_capacity(keys.len() + 1); - del_keys.push(&resolved); - del_keys.extend(keys.iter().map(String::as_str)); - self.store.del(&del_keys); + let del_keys: Vec<&str> = keys.iter().map(String::as_str).collect(); + if !del_keys.is_empty() { + self.store.del(&del_keys); + } Ok(()) } diff --git a/src/fs/kvspace.rs b/src/fs/kvspace.rs index ab5df7f..f1c9888 100644 --- a/src/fs/kvspace.rs +++ b/src/fs/kvspace.rs @@ -11,7 +11,7 @@ use crate::coord::cmp_coord; use crate::kvspace::{KVPair, KVSpace}; use crate::kvspace_common::{ is_descendant, is_frame_operand_key, join_path, sep_path, split_index, strip_dir_suf, - validate_ptr, watch_value, + subtree_base, validate_ptr, watch_value, }; use crate::r#const::*; use crate::xvalue::*; @@ -611,32 +611,67 @@ impl KVSpace for FsKVSpace { if crate::r#const::is_reserved(prefix) { return Err(format!("reserved metadata key: {prefix}")); } - let resolved = self.resolve_path(prefix); + // Resolve links only on parent components (POSIX rm semantics); if the final + // component is itself a link, delete the link rather than its target subtree. + let resolved = self.resolve_parent(prefix); if crate::r#const::is_reserved(&resolved) { return Err(format!("reserved metadata key: {resolved}")); } - // 若 prefix 本身是链接（叶 Ptr），只删链接。 - let link_key = if Self::is_dir_key(&resolved) && resolved != PATH_SEP { - strip_dir_suf(&resolved) - } else { - &resolved - }; - if let Some(data) = self.read_leaf(link_key) { - if from_wire_head(&data).is_ptr() { - return self.del(&[resolved]); + let base = subtree_base(&resolved).to_string(); + if !base.is_empty() { + if let Some(data) = self.read_leaf(&base) { + if from_wire_head(&data).is_ptr() { + return self.del(&[base]); + } } } - let base = if Self::is_dir_key(&resolved) && resolved != PATH_SEP { - strip_dir_suf(&resolved) + // Subtree covers base, base/*, base·*, and base‥* across the node path, its + // sibling `base·` directory, and sibling `base‥*` entries in the parent directory. + let mut doomed = Vec::new(); + if base.is_empty() { + // Root: remove everything except reserved metadata, keeping the root dir itself. + if let Ok(entries) = fs::read_dir(&self.root) { + for e in entries.flatten() { + if e.file_name() != META_ROOT_NAME { + doomed.push(e.path()); + } + } + } } else { - &resolved - }; - for key in [base.to_string(), format!("{base}{OBJ_SEP}")] { - let node = self.node_path(&key); - if node.is_dir() { - fs::remove_dir_all(&node).map_err(|e| format!("DelTree {key}: {e}"))?; + let node = self.node_path(&base); + doomed.push(self.node_path(&format!("{base}{OBJ_SEP}"))); + if let (Some(parent), Some(name)) = (node.parent(), node.file_name()) { + let stem = format!( + "{}{}", + Self::unescape_name(&name.to_string_lossy()), + RUNTIME_MEMBER_SEP + ); + if let Ok(entries) = fs::read_dir(parent) { + for e in entries.flatten() { + if !is_tmp_name(&e.file_name()) + && Self::unescape_name(&e.file_name().to_string_lossy()) + .starts_with(&stem) + { + doomed.push(e.path()); + } + } + } + } + doomed.push(node); + } + for node in doomed { + let r = if node.is_dir() { + fs::remove_dir_all(&node) } else if node.is_file() { - fs::remove_file(&node).map_err(|e| format!("DelTree {key}: {e}"))?; + fs::remove_file(&node) + } else { + continue; + }; + match r { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("DelTree {}: {e}", node.display())); + } + _ => {} } } Ok(()) diff --git a/src/kvspace_common.rs b/src/kvspace_common.rs index 0bc0aac..acde0c8 100644 --- a/src/kvspace_common.rs +++ b/src/kvspace_common.rs @@ -28,6 +28,15 @@ pub fn strip_dir_suf(path: &str) -> &str { } } +/// Subtree root for DelTree: strips a single trailing `/` or `·` (`"/"` maps to `""`). +/// Deletes every key satisfying `store::is_subtree_key(base, key)`. +pub fn subtree_base(prefix: &str) -> &str { + if prefix == PATH_SEP { + "" + } else { + strip_dir_suf(prefix) + } +} pub fn is_descendant(base: &str, path: &str) -> bool { if base == PATH_SEP { return path != PATH_SEP && path.starts_with(PATH_SEP); diff --git a/tests/deltree.rs b/tests/deltree.rs new file mode 100644 index 0000000..866a316 --- /dev/null +++ b/tests/deltree.rs @@ -0,0 +1,277 @@ +// deltree.rs - kvspaceDelTree deletion semantics across fs and redis (C ABI). +// +// Deletion scope matches kvspace-c shm and store::is_subtree_key: `base` itself plus every +// descendant separated by `/`, `·`, or `‥` (including hidden `‥` runtime keys). A trailing +// `/` or `·` on `prefix` normalizes to `base`. If the final path component is a link, only +// the link is deleted (POSIX `rm` semantics); links on parent components are resolved. +// +// Redis tests skip when 127.0.0.1:6379 is unreachable and isolate under a unique prefix. +use kvspace_durable::ffi::*; +use std::ffi::CString; + +fn xvalue_i64(n: i64) -> Vec { + let mut v = vec![0u8; 16]; + v[0..6].copy_from_slice(&8u64.to_le_bytes()[..6]); + v[8..16].copy_from_slice(&n.to_le_bytes()); + v.extend_from_slice(&[2, 0]); // empty attr + v +} + +unsafe fn set_raw(h: *mut Handle, key: &str, v: &[u8]) { + let k = CString::new(key).unwrap(); + let mut err = [0 as std::ffi::c_char; 256]; + let rc = kvspaceSetValue( + h, + k.as_ptr(), + v.as_ptr(), + v.len() as u32, + err.as_mut_ptr(), + 256, + ); + let msg = std::ffi::CStr::from_ptr(err.as_ptr()) + .to_string_lossy() + .into_owned(); + assert_eq!(rc, 0, "set {key}: {msg}"); +} + +unsafe fn set(h: *mut Handle, key: &str, n: i64) { + set_raw(h, key, &xvalue_i64(n)); +} + +unsafe fn exists(h: *mut Handle, key: &str) -> bool { + let k = CString::new(key).unwrap(); + let mut p: *mut u8 = std::ptr::null_mut(); + let mut l = 0u32; + kvspaceGet(h, k.as_ptr(), 0, &mut p, &mut l); + !p.is_null() && l > 0 +} + +unsafe fn deltree(h: *mut Handle, prefix: &str) { + let p = CString::new(prefix).unwrap(); + let mut err = [0 as std::ffi::c_char; 256]; + let rc = kvspaceDelTree(h, p.as_ptr(), err.as_mut_ptr(), 256); + let msg = std::ffi::CStr::from_ptr(err.as_ptr()) + .to_string_lossy() + .into_owned(); + assert_eq!(rc, 0, "DelTree({prefix}): {msg}"); +} + +unsafe fn list_len(h: *mut Handle, dir: &str) -> i32 { + let d = CString::new(dir).unwrap(); + let mut n = -1; + kvspaceListLen(h, d.as_ptr(), 0, 0, &mut n); + n +} + +/// `root` is a top-level prefix exclusive to this test run (e.g. `"/dt123"`). +unsafe fn run(dsn: &str, root: &str) { + let c = CString::new(dsn).unwrap(); + let h = kvspaceConnect(c.as_ptr()); + let h2 = kvspaceConnect(c.as_ptr()); + assert!(!h.is_null() && !h2.is_null(), "connect {dsn}"); + let d = format!("{root}/d"); + let inside: Vec = [ + "", + "/a", + "/\u{2025}status", + "/sub/x", + "/sub/\u{2025}pc", + "\u{b7}k", + "\u{b7}k/y", + "\u{b7}k\u{2025}x", + "\u{2025}x", + "\u{2025}x/y", + ] + .iter() + .map(|s| format!("{d}{s}")) + .collect(); + let outside: Vec = ["x", "2/a", "2\u{2025}pc"] + .iter() + .map(|s| format!("{d}{s}")) + .chain([format!("{root}/e/\u{2025}status")]) + .collect(); + for form in [d.clone(), format!("{d}/"), format!("{d}\u{b7}")] { + let mk = CString::new(format!("{d}/")).unwrap(); + assert_eq!( + kvspaceMkindex(h, mk.as_ptr(), 0, std::ptr::null_mut(), 0), + 0 + ); + for (i, k) in inside.iter().chain(outside.iter()).enumerate() { + set(h, k, i as i64); + } + for k in inside.iter().chain(outside.iter()) { + assert!(exists(h, k), "{dsn}: seeded {k}"); + } + // Populate list snapshots on both handles; DelTree must invalidate/refresh them. + assert_eq!(list_len(h, &format!("{d}/")), 2, "{dsn}: /d/ lists a, sub/"); + assert_eq!( + list_len(h2, &format!("{d}/sub/")), + 1, + "{dsn}: /d/sub/ lists x" + ); + deltree(h, &form); + for k in &inside { + assert!(!exists(h, k), "{dsn}: DelTree({form}) left {k}"); + assert!( + !exists(h2, k), + "{dsn}: DelTree({form}) left {k} (other handle)" + ); + } + assert!( + !exists(h, &format!("{d}/")), + "{dsn}: DelTree({form}) left {d}/" + ); + for k in &outside { + assert!(exists(h, k), "{dsn}: DelTree({form}) removed sibling {k}"); + } + assert_eq!( + list_len(h, &format!("{d}/")), + 0, + "{dsn}: list after DelTree({form})" + ); + assert_eq!( + list_len(h2, &format!("{d}/sub/")), + 0, + "{dsn}: other handle after DelTree" + ); + } + + // Link final component: delete only the link itself (with or without trailing '/'), leaving target intact. + let t = format!("{root}/t"); + set(h, &format!("{t}/a"), 1); + set(h, &format!("{t}/\u{2025}pc"), 2); + set(h, &format!("{t}\u{2025}x"), 3); + let tc = CString::new(t.clone()).unwrap(); + let empty = CString::new("").unwrap(); + let mut lv: *mut u8 = std::ptr::null_mut(); + let mut ln = 0u32; + assert_eq!( + kvspaceNewPtr(empty.as_ptr(), tc.as_ptr(), &mut lv, &mut ln), + 0 + ); + let link = std::slice::from_raw_parts(lv, ln as usize).to_vec(); + // Reclaim the Box<[u8]> leaked by kvspaceNewPtr within the same Rust allocator. + drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( + lv, + ln as usize, + ))); + for (key, form) in [("p", "p"), ("q", "q/")] { + let lk = format!("{root}/{key}"); + set_raw(h, &lk, &link); + deltree(h, &format!("{root}/{form}")); + assert!(!exists(h, &lk), "{dsn}: DelTree({form}) left the link"); + for k in [ + format!("{t}/a"), + format!("{t}/\u{2025}pc"), + format!("{t}\u{2025}x"), + ] { + assert!( + exists(h, &k), + "{dsn}: DelTree(link {form}) removed target key {k}" + ); + } + } + + // Resolve links on parent components without following the final link: /p2 -> /t2, /t2/q -> /t; + // DelTree("/p2/q") removes only /t2/q. + let t2 = CString::new(format!("{root}/t2")).unwrap(); + let mut l2: *mut u8 = std::ptr::null_mut(); + let mut n2 = 0u32; + assert_eq!( + kvspaceNewPtr(empty.as_ptr(), t2.as_ptr(), &mut l2, &mut n2), + 0 + ); + let link2 = std::slice::from_raw_parts(l2, n2 as usize).to_vec(); + drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut( + l2, + n2 as usize, + ))); + set(h, &format!("{root}/t2/keep"), 5); + set_raw(h, &format!("{root}/t2/q"), &link); + set_raw(h, &format!("{root}/p2"), &link2); + deltree(h, &format!("{root}/p2/q")); + assert!( + !exists(h, &format!("{root}/t2/q")), + "{dsn}: DelTree via parent link left the link" + ); + assert!( + exists(h, &format!("{root}/t2/keep")), + "{dsn}: DelTree via parent link removed a sibling" + ); + assert!( + exists(h, &format!("{root}/p2")), + "{dsn}: DelTree via parent link removed the parent link" + ); + for k in [ + format!("{t}/a"), + format!("{t}/\u{2025}pc"), + format!("{t}\u{2025}x"), + ] { + assert!( + exists(h, &k), + "{dsn}: DelTree via parent link removed target key {k}" + ); + } + + deltree(h, root); + for k in outside + .iter() + .chain([format!("{t}/a"), format!("{t}\u{2025}x")].iter()) + { + assert!(!exists(h, k), "{dsn}: cleanup left {k}"); + } + kvspaceClose(h2); + kvspaceClose(h); +} + +#[test] +fn deltree_fs() { + let dir = std::env::temp_dir().join(format!("kvspace-deltree-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let dsn = format!("fs://{}", dir.display()); + unsafe { + run(&dsn, "/r"); + // Root "/": delete all keys (including `‥` keys) while keeping the root directory usable. + let c = CString::new(dsn.as_str()).unwrap(); + let h = kvspaceConnect(c.as_ptr()); + set(h, "/a/b", 1); + set(h, "/a\u{2025}x", 2); + set(h, "/\u{2025}top", 3); + deltree(h, "/"); + for k in ["/a/b", "/a\u{2025}x", "/\u{2025}top"] { + assert!(!exists(h, k), "DelTree(/) left {k}"); + } + set(h, "/after", 4); + assert!(exists(h, "/after"), "store usable after DelTree(/)"); + kvspaceClose(h); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn deltree_redis() { + let dsn = std::env::var("KVSPACE_TEST_REDIS_DSN") + .unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string()); + let addr = dsn + .trim_start_matches("redis://") + .split('/') + .next() + .unwrap_or("") + .to_string(); + let reachable = std::net::ToSocketAddrs::to_socket_addrs(&addr.as_str()) + .ok() + .and_then(|mut a| a.next()) + .is_some_and(|a| { + std::net::TcpStream::connect_timeout(&a, std::time::Duration::from_millis(300)).is_ok() + }); + if !reachable { + eprintln!("skipping redis DelTree test: {addr} unreachable"); + return; + } + unsafe { + run( + &dsn, + &format!("/kvspace-deltree-test-{}", std::process::id()), + ) + }; +} From c4400cd52d7c9e441c0b216082b79ea4319f95eb Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 14:36:41 +0000 Subject: [PATCH 06/12] ci: pin bundled kvspace ABI headers in release workflow via deps.json Add deps.json (kvspace v0.3.23) and clone kvspace at that tag when packaging release headers instead of tracking default branch HEAD. --- .github/workflows/release.yml | 6 ++++-- deps.json | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 deps.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af87c04..ca8e46d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,8 +19,10 @@ jobs: v="${GITHUB_REF_NAME}" d="kvspace-durable-abi-$v-linux-x86_64" mkdir -p "$d/include" "$d/lib" - # 头：本仓为 Rust（无 C 头），权威 ABI 头在 kvspace 仓 → 递归整目录 - git clone --depth 1 https://github.com/array2d/kvspace.git /tmp/kvspace-hdr + # Bundle ABI headers pinned to the kvspace tag in deps.json. + kvspace_ver=$(jq -r '.kvspace' deps.json) + test -n "$kvspace_ver" && test "$kvspace_ver" != null + git clone --depth 1 --branch "$kvspace_ver" https://github.com/array2d/kvspace.git /tmp/kvspace-hdr cp -R /tmp/kvspace-hdr/include/. "$d/include/" # 库：扫 target/release 的 .so*/.dylib（-P 保符号链接） find target/release -maxdepth 2 \( -name 'lib*.so*' -o -name 'lib*.dylib' \) -exec cp -P {} "$d/lib/" \; diff --git a/deps.json b/deps.json new file mode 100644 index 0000000..16fac46 --- /dev/null +++ b/deps.json @@ -0,0 +1,3 @@ +{ + "kvspace": "v0.3.23" +} From 555709a8db2943522358901a905db0247971fde9 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 18:22:42 +0000 Subject: [PATCH 07/12] kvspace-durable: remove SetValue validate_ptr check and preserve untyped ST_SLACK values - backend, fs: skip validate_ptr during set to support forward, multi-hop, and cyclic pointers without extra synchronous reads; validate attr_len >= 2 and trim trailing padding in kvspaceSetValue - kvspace_common: preserve map key shapes in norm_langtype - xvalue: decode to XValue::None only for canonical ST_SHORT + REF_INLINE + len=0 + empty langtype, and map empty kind to ST_SLACK in to_wire_head so untyped system slots round-trip and work with Watch --- src/backend.rs | 16 ++++++++++------ src/fs/kvspace.rs | 16 ++++++++++------ src/kvspace_common.rs | 3 +++ src/xvalue.rs | 16 +++++++++++++--- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/backend.rs b/src/backend.rs index 6414df1..a661afa 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -6,7 +6,7 @@ use std::time::Duration; use crate::kvspace::{KVPair, KVSpace}; use crate::kvspace_common::{ is_descendant, is_frame_operand_key, join_path, sep_path, split_index, strip_dir_suf, - subtree_base, validate_ptr, watch_value, + subtree_base, watch_value, }; use crate::r#const::*; use crate::store::KVStore; @@ -329,13 +329,17 @@ impl KVSpace for Backend { .ok_or_else(|| format!("invalid XValue at {key}"))?; let attr = crate::headlenpow::attr_len(&raw, head.len as usize) .ok_or_else(|| format!("invalid XValue at {key}"))?; - if head.len as usize + crate::headlenpow::PREFIX + attr as usize != raw.len() { + if attr < 2 { return Err(format!("invalid XValue at {key}")); } - // 指针校验用声明的 target_kindexpr（val 为 Ptr 时可得；raw 直写无类型则跳过）。 - if let XValue::Ptr(ptr) = &pair.val { - validate_ptr(self, &ptr.target, &ptr.target_kindexpr)?; + let total = (head.len as usize) + .checked_add(crate::headlenpow::PREFIX) + .and_then(|x| x.checked_add(attr as usize)) + .ok_or_else(|| format!("invalid XValue at {key}"))?; + if total > raw.len() { + return Err(format!("invalid XValue at {key}")); } + let raw = &raw[..total]; let (parent, _, _) = split_index(&key); if parent.ends_with(OBJ_SEP) { let base = strip_dir_suf(&parent); @@ -356,7 +360,7 @@ impl KVSpace for Backend { } } // 单 key 整体写（类型在值 attr 区）：天然原子，无 attr key、无写序。 - self.store.set(&key, &raw); + self.store.set(&key, raw); } Ok(()) } diff --git a/src/fs/kvspace.rs b/src/fs/kvspace.rs index f1c9888..cba4eb2 100644 --- a/src/fs/kvspace.rs +++ b/src/fs/kvspace.rs @@ -11,7 +11,7 @@ use crate::coord::cmp_coord; use crate::kvspace::{KVPair, KVSpace}; use crate::kvspace_common::{ is_descendant, is_frame_operand_key, join_path, sep_path, split_index, strip_dir_suf, - subtree_base, validate_ptr, watch_value, + subtree_base, watch_value, }; use crate::r#const::*; use crate::xvalue::*; @@ -524,13 +524,17 @@ impl KVSpace for FsKVSpace { .ok_or_else(|| format!("invalid XValue at {key}"))?; let attr = crate::headlenpow::attr_len(&raw, head.len as usize) .ok_or_else(|| format!("invalid XValue at {key}"))?; - if head.len as usize + crate::headlenpow::PREFIX + attr as usize != raw.len() { + if attr < 2 { return Err(format!("invalid XValue at {key}")); } - // 指针校验用声明的 target_kindexpr（val 为 Ptr 时可得；raw 直写无类型则跳过）。 - if let XValue::Ptr(ptr) = &pair.val { - validate_ptr(self, &ptr.target, &ptr.target_kindexpr)?; + let total = (head.len as usize) + .checked_add(crate::headlenpow::PREFIX) + .and_then(|x| x.checked_add(attr as usize)) + .ok_or_else(|| format!("invalid XValue at {key}"))?; + if total > raw.len() { + return Err(format!("invalid XValue at {key}")); } + let raw = &raw[..total]; let (parent, _, _) = split_index(&key); if parent.ends_with(OBJ_SEP) { let base = strip_dir_suf(&parent); @@ -555,7 +559,7 @@ impl KVSpace for FsKVSpace { if Self::is_dir_key(&key) { self.ensure_dir(&key); } - self.write_leaf(&key, &raw); + self.write_leaf(&key, raw); } Ok(()) } diff --git a/src/kvspace_common.rs b/src/kvspace_common.rs index acde0c8..998ddd2 100644 --- a/src/kvspace_common.rs +++ b/src/kvspace_common.rs @@ -108,6 +108,9 @@ pub fn split_index(path: &str) -> (String, String, SepKind) { /// （见 [[map容器]]：`[float64,float64]` ≠ `[2]float64`），一律原样保留——剥掉会让 map 的 /// 键类型整个丢失，Ptr 类型检查随即误判。 fn norm_langtype(s: &str) -> String { + if s.contains(OBJ_SEP) { + return s.to_string(); + } if let Some(rest) = s.strip_prefix('[') { if let Some(end) = rest.find(']') { let inner = &rest[..end]; diff --git a/src/xvalue.rs b/src/xvalue.rs index ba6c99f..b2243d7 100644 --- a/src/xvalue.rs +++ b/src/xvalue.rs @@ -430,13 +430,15 @@ fn array_to_header(kind: &str, array_len: i32) -> Vec { /// storetype 判定（对齐 kvspace 三正交轴与 C codec）： /// - ref==1（指针）→ ST_SLACK；ref==2（@ext）→ ST_EXT； /// - rwir/def langtype 或带 body 的 rwfunc（代码槽）→ ST_SLACK； +/// - untyped variable-length system slots (empty kind, e.g. `‥lib`/`‥status`/`‥pc`) -> ST_SLACK; /// - char/*（字符/字节数组）→ ST_SLACK； /// - dims 非空且非 map/struct → ST_LARGE（张量）； /// - 否则 ST_SHORT（None/标量/目录/map 头/空 rwfunc 目录值）。 pub fn to_wire_head(kind: &str, r#ref: i32, dims: &[i32], raw: &[u8]) -> Vec { let (storetype, ptr) = match r#ref { 0 => { - let slack = matches!(kind, "rwir" | "def langtype") + let slack = kind.is_empty() + || matches!(kind, "rwir" | "def langtype") || (kind == "rwfunc" && !raw.is_empty()) || kind.starts_with("char/"); let st = if slack { @@ -492,8 +494,16 @@ pub fn from_wire_head(data: &[u8]) -> XValueHead { /// 从完整 XValue 字节（固定 head + body + attr）组装 XValue。 pub fn from_wire(data: &[u8]) -> XValue { let h = from_wire_head(data); - // 空 langtype 表示 None（无类型 = 空 attr）。 - if h.langtype.is_empty() && !h.is_ptr() && h.r#ref() != REF_EXT as i32 { + if h.headlen == 0 { + return XValue::None; + } + // Canonical None is ST_SHORT + REF_INLINE + empty body + empty langtype; + // untyped system slots (`‥lib`/`‥status`/`‥pc`) use ST_SLACK and must keep their body. + if h.langtype.is_empty() + && h.r#ref == REF_INLINE + && h.storetype == ST_SHORT + && h.body_len == 0 + { return XValue::None; } h.from_wire(h.body(data)) From 418d3ce39869584eaea2ee04d67501f048f88015 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 18:22:42 +0000 Subject: [PATCH 08/12] ffi: resolve leaf Ptr chains up to 16 hops and share read_cache in GetHead - kvspaceGet, kvspaceWriteInPlace: follow leaf Ptr chains up to 16 hops when resolve != 0 (returning 1 on cycle/hop limit and 0 on dangling target), caching hops so raw and resolved borrowed views coexist - kvspaceGetHead, kvspaceGetPart, kvspaceSetPart: resolve leaf Ptr chains through read_cache and invalidate the terminal target key on mutation --- src/ffi.rs | 340 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 207 insertions(+), 133 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 3b81b2f..e100f35 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -86,7 +86,7 @@ impl Handle { raw: Some(xv), }]) }); - self.read_cache.remove(&key); + self.cache_invalidate(&key); r?; } Ok(()) @@ -190,59 +190,6 @@ fn volatile_refresh(hd: &mut Handle, key: &str) -> bool { true } -fn volatile_get(hd: &mut Handle, key: &str, out: *mut *mut u8, out_len: *mut u32) -> c_int { - if !volatile_refresh(hd, key) { - unsafe { - *out = std::ptr::null_mut(); - *out_len = 0; - } - return 1; - } - let vb = hd.volatile_cache.get_mut(key).unwrap(); - let b = &mut vb.slots[vb.which]; - unsafe { - if b.is_empty() { - *out = std::ptr::null_mut(); - *out_len = 0; - } else { - *out = b.as_mut_ptr(); - *out_len = b.len() as u32; - } - } - 0 -} - -fn volatile_get_part( - hd: &mut Handle, - key: &str, - offset: u32, - len: u32, - out: *mut *mut u8, - out_len: *mut u32, -) -> c_int { - if !volatile_refresh(hd, key) { - unsafe { - *out = std::ptr::null_mut(); - *out_len = 0; - } - return 1; - } - let vb = hd.volatile_cache.get_mut(key).unwrap(); - let b = &mut vb.slots[vb.which]; - let s = (offset as usize).min(b.len()); - let e = (s + len as usize).min(b.len()); - unsafe { - if e <= s { - *out = std::ptr::null_mut(); - *out_len = 0; - } else { - *out = b.as_mut_ptr().add(s); - *out_len = (e - s) as u32; - } - } - 0 -} - /// flush 上一笔惰性写后返回后端引用。任一步失败 → Err（调用方转 err/err 返回 1）。 /// Used by mutating structure ops (`Del`/`DelTree`/`Cp*`/`Mkindex*`/`RmindexExt`/`Clear`); clears `list_cache`. unsafe fn kv_flush<'a>(h: *mut Handle) -> Result<&'a mut dyn KVSpace, String> { @@ -410,13 +357,97 @@ pub extern "C" fn kvspaceClose(h: *mut Handle) { // ── KVSpace 原语 ───────────────────────────────────────────────────── -/// 借用读：*out 指向读借用池内本笔缓冲（活到下一次写 flush），调用方不得 free。 -/// resolve 由 get_raw 内部按路径解析（durable 恒穿透父路径 link）；空值 → *out=NULL、out_len=0。 +/// Fetches raw bytes for a single key (using `read_cache`, or `volatile_refresh` for control keys). +/// Returns `Ok(true)` if present and non-empty, `Ok(false)` if missing/empty, or `Err(())` on backend failure. +fn fetch_raw_one(hd: &mut Handle, key: &str) -> Result { + if is_volatile(key) { + if !volatile_refresh(hd, key) { + return Err(()); + } + let vb = hd.volatile_cache.get(key).unwrap(); + return Ok(!vb.slots[vb.which].is_empty()); + } + if hd.read_cache.contains_key(key) { + return Ok(true); + } + let raw = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(key))) + .map_err(|_| ())?; + if raw.is_empty() { + return Ok(false); + } + hd.read_cache.insert(key.to_string(), raw); + Ok(true) +} + +fn cached_slice<'a>(hd: &'a Handle, key: &str) -> Option<&'a [u8]> { + if is_volatile(key) { + let vb = hd.volatile_cache.get(key)?; + let s = &vb.slots[vb.which]; + (!s.is_empty()).then_some(s.as_slice()) + } else { + hd.read_cache.get(key).map(|v| v.as_slice()) + } +} + +fn cached_mut_slice<'a>(hd: &'a mut Handle, key: &str) -> Option<&'a mut [u8]> { + if is_volatile(key) { + let vb = hd.volatile_cache.get_mut(key)?; + let s = &mut vb.slots[vb.which]; + (!s.is_empty()).then_some(s.as_mut_slice()) + } else { + hd.read_cache.get_mut(key).map(|v| v.as_mut_slice()) + } +} + +/// Maximum leaf `Ptr` hops to follow (matches `kvspace-c`). +const MAX_PTR_HOPS: usize = 16; + +/// Resolves `key` in cache (`resolve = true` follows leaf `Ptr` chains up to 16 hops). +/// Returns `Ok(Some(final_key))` if found, `Ok(None)` if missing/dangling, or `Err(())` on error/cycle/hop limit. +fn resolve_cached_key(hd: &mut Handle, key: &str, resolve: bool) -> Result, ()> { + if !resolve { + return match fetch_raw_one(hd, key)? { + true => Ok(Some(key.to_string())), + false => Ok(None), + }; + } + let mut cur = key.to_string(); + for hop in 0..=MAX_PTR_HOPS { + if cur.is_empty() { + return Ok(None); + } + if !fetch_raw_one(hd, &cur)? { + return Ok(None); + } + let next_target = { + let bytes = cached_slice(hd, &cur).ok_or(())?; + let w = crate::headlenpow::from_wire(bytes).ok_or(())?; + if w.ptr != REF_PTR { + None + } else { + if hop == MAX_PTR_HOPS { + return Err(()); + } + let t = std::str::from_utf8(w.body).map_err(|_| ())?; + Some(t.to_string()) + } + }; + match next_target { + None => return Ok(Some(cur)), + Some(t) => cur = t, + } + } + Err(()) +} + +/// Borrowed read: `*out` points into the handle's cache (valid until the key is overwritten, +/// deleted, or the handle is closed; caller must not `free`). `resolve != 0` follows leaf `Ptr` +/// chains up to 16 hops (cycle/limit returns 1; missing/dangling sets `*out = NULL`, `*out_len = 0`, returns 0). #[no_mangle] pub extern "C" fn kvspaceGet( h: *mut Handle, key: *const c_char, - _resolve: c_int, + resolve: c_int, out: *mut *mut u8, out_len: *mut u32, ) -> c_int { @@ -432,36 +463,29 @@ pub extern "C" fn kvspaceGet( } return 1; } - /* 控制键：每次重读后端，写进双缓冲的另一块；上一块保活到本 key 下一次读。 */ - if is_volatile(&key) { - return volatile_get(hd, &key, out, out_len); - } - /* 命中：返回同一份常驻快照。绝不重读/替换——同指令两次读同一 key 的视图必须共存。 */ - if let Some(b) = hd.read_cache.get_mut(&key) { - unsafe { - *out = b.as_mut_ptr(); - *out_len = b.len() as u32; - } - return 0; - } - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(&key))) { - Ok(raw) => { - if raw.is_empty() { + match resolve_cached_key(hd, &key, resolve != 0) { + Ok(Some(final_key)) => { + if let Some(b) = cached_mut_slice(hd, &final_key) { unsafe { - *out = std::ptr::null_mut(); - *out_len = 0; + *out = b.as_mut_ptr(); + *out_len = b.len() as u32; } } else { - hd.read_cache.insert(key.clone(), raw); - let b = hd.read_cache.get_mut(&key).unwrap(); unsafe { - *out = b.as_mut_ptr(); - *out_len = b.len() as u32; + *out = std::ptr::null_mut(); + *out_len = 0; } } 0 } - Err(_) => { + Ok(None) => { + unsafe { + *out = std::ptr::null_mut(); + *out_len = 0; + } + 0 + } + Err(()) => { unsafe { *out = std::ptr::null_mut(); *out_len = 0; @@ -493,11 +517,26 @@ pub extern "C" fn kvspaceSetValue( } let key = unsafe { cstr(key) }.to_string(); let raw = unsafe { std::slice::from_raw_parts(value, value_len as usize) }; - if crate::headlenpow::from_wire(raw).is_none() { + let Some(whead) = crate::headlenpow::from_wire(raw) else { + write_err(err, err_cap, "invalid XValue"); + return 1; + }; + let Some(attr) = crate::headlenpow::attr_len(raw, whead.len as usize) else { + write_err(err, err_cap, "invalid XValue"); + return 1; + }; + let Some(total) = (whead.len as usize) + .checked_add(crate::headlenpow::PREFIX) + .and_then(|x| x.checked_add(attr as usize)) + else { + write_err(err, err_cap, "invalid XValue"); + return 1; + }; + if attr < 2 || total > raw.len() { write_err(err, err_cap, "invalid XValue"); return 1; } - let raw = raw.to_vec(); + let raw = raw[..total].to_vec(); hd.list_cache = None; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let val = from_wire(&raw); @@ -512,7 +551,7 @@ pub extern "C" fn kvspaceSetValue( let code = result_to_code(result, err, err_cap); if code == 0 { /* 写后失效：不把写值留在读缓存里，避免写密集负载无界占用。 */ - hd.read_cache.remove(&key); + hd.cache_invalidate(&key); } code } @@ -524,7 +563,7 @@ pub extern "C" fn kvspaceSetValue( } /// 定位读：借用读 key 值的 [offset, offset+len) 字节，*out 指向该 key 读缓存（活到该 key 被写/删或 close，调用方不得 free）。 -/// 空/不存在/越界为空 → *out=NULL、out_len=0。 +/// Always follows leaf `Ptr` chains; sets `*out = NULL` and `*out_len = 0` when missing, empty, or out of range. #[no_mangle] pub extern "C" fn kvspaceGetPart( h: *mut Handle, @@ -546,34 +585,32 @@ pub extern "C" fn kvspaceGetPart( } return 1; } - if is_volatile(&key) { - return volatile_get_part(hd, &key, offset, len, out, out_len); - } - /* 缓存的是整值：GetPart 未命中先取整值再切片（与后端 get_part 同字节）。 */ - if !hd.read_cache.contains_key(&key) { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(&key))) { - Ok(raw) => { - if raw.is_empty() { - unsafe { - *out = std::ptr::null_mut(); - *out_len = 0; - } - return 0; - } - hd.read_cache.insert(key.clone(), raw); + let final_key = match resolve_cached_key(hd, &key, true) { + Ok(Some(k)) => k, + Ok(None) => { + unsafe { + *out = std::ptr::null_mut(); + *out_len = 0; } - Err(_) => { - unsafe { - *out = std::ptr::null_mut(); - *out_len = 0; - } - return 1; + return 0; + } + Err(()) => { + unsafe { + *out = std::ptr::null_mut(); + *out_len = 0; } + return 1; } - } - let b = hd.read_cache.get_mut(&key).unwrap(); + }; + let Some(b) = cached_mut_slice(hd, &final_key) else { + unsafe { + *out = std::ptr::null_mut(); + *out_len = 0; + } + return 0; + }; let s = (offset as usize).min(b.len()); - let e = (s + len as usize).min(b.len()); + let e = s.saturating_add(len as usize).min(b.len()); unsafe { if e <= s { *out = std::ptr::null_mut(); @@ -586,7 +623,7 @@ pub extern "C" fn kvspaceGetPart( 0 } -/// 定位写：就地写 buf 到 key 值的 [offset, offset+buf_len)（key 须已存在、不改结构）。立即持久，非惰性。 +/// Positional write into `[offset, offset + buf_len)` of an existing key (follows leaf `Ptr` chains; synchronous). #[no_mangle] pub extern "C" fn kvspaceSetPart( h: *mut Handle, @@ -601,26 +638,46 @@ pub extern "C" fn kvspaceSetPart( Some(x) => x, None => return 1, }; + if buf_len > 0 && buf.is_null() { + write_err(err, err_cap, "invalid SetPart buffer"); + return 1; + } if let Err(e) = hd.flush() { write_err(err, err_cap, &e); return 1; } let key = unsafe { cstr(key) }.to_string(); - let data = unsafe { std::slice::from_raw_parts(buf, buf_len as usize) }; + let target_key = match resolve_cached_key(hd, &key, true) { + Ok(Some(k)) => k, + Ok(None) => { + write_err(err, err_cap, &format!("set_part: missing key {key}")); + return 1; + } + Err(()) => { + write_err(err, err_cap, &format!("set_part: ptr resolve failed at {key}")); + return 1; + } + }; + let data: &[u8] = if buf_len == 0 { + &[] + } else { + unsafe { std::slice::from_raw_parts(buf, buf_len as usize) } + }; hd.list_cache = None; let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - hd.kv.set_part(&key, offset, data) + hd.kv.set_part(&target_key, offset, data) })) .map_err(panic_msg) .and_then(|x| x); let code = result_to_code(r, err, err_cap); if code == 0 { - hd.read_cache.remove(&key); + hd.cache_invalidate(&target_key); } code } -/// 读 head：只读值前缀并解码三正交轴 head（不取 body）。空/不存在 → 返回 1。 +/// Decodes the value header using `read_cache` and following leaf `Ptr` chains (up to 16 hops). +/// Returns 1 when missing, empty, dangling, or exceeding the hop limit. #[no_mangle] pub extern "C" fn kvspaceGetHead( h: *mut Handle, @@ -635,16 +692,15 @@ pub extern "C" fn kvspaceGetHead( return 1; } let key = unsafe { cstr(key) }.to_string(); - let value = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(&key))) - { - Ok(p) => p, - Err(_) => return 1, + let final_key = match resolve_cached_key(hd, &key, true) { + Ok(Some(k)) => k, + _ => return 1, }; - if value.is_empty() { + let Some(value) = cached_slice(hd, &final_key) else { return 1; - } + }; /* langtype 在值 attr 区（单 key 整体）；无类型 = 空 attr。 */ - let head = crate::xvalue::from_wire_head(&value); + let head = crate::xvalue::from_wire_head(value); if head.headlen == 0 { return 1; } @@ -654,11 +710,12 @@ pub extern "C" fn kvspaceGetHead( /// 就地写：key 必须已存在、body_len 必须等于原 body_len——把原 head+body 攒进 write_buf、 /// 置 pending，返回 write_buf 内 body 偏移指针供直接写；违反前置条件 → 非 0 + err。 +/// When `resolve != 0`, follows leaf `Ptr` chains (up to 16 hops) to mutate the terminal target in place. #[no_mangle] pub extern "C" fn kvspaceWriteInPlace( h: *mut Handle, key: *const c_char, - _resolve: c_int, + resolve: c_int, body_len: u32, body: *mut *mut u8, err: *mut c_char, @@ -668,23 +725,39 @@ pub extern "C" fn kvspaceWriteInPlace( Some(x) => x, None => return 1, }; + if body.is_null() { + write_err(err, err_cap, "kvspace: null body out-pointer"); + return 1; + } if let Err(e) = hd.flush() { write_err(err, err_cap, &e); return 1; } let key = unsafe { cstr(key) }.to_string(); - let existing = - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(&key))) { - Ok(v) => v, - Err(p) => { - write_err(err, err_cap, &panic_msg(p)); + let target_key = match resolve_cached_key(hd, &key, resolve != 0) { + Ok(Some(k)) => k, + Ok(None) => { + write_err(err, err_cap, "kvspace: write-in-place on missing key"); + return 1; + } + Err(()) => { + write_err(err, err_cap, "kvspace: write-in-place ptr resolve failed"); + return 1; + } + }; + let existing = match hd.read_cache.remove(&target_key) { + Some(v) => v, + None => match cached_slice(hd, &target_key).map(|s| s.to_vec()) { + Some(v) => { + hd.volatile_cache.remove(&target_key); + v + } + None => { + write_err(err, err_cap, "kvspace: write-in-place on missing key"); return 1; } - }; - if existing.is_empty() { - write_err(err, err_cap, "kvspace: write-in-place on missing key"); - return 1; - } + }, + }; let head = from_wire_head(&existing); let head_len = head.head_len() as usize; let blen = head.body_len.max(0) as usize; @@ -692,14 +765,15 @@ pub extern "C" fn kvspaceWriteInPlace( if head.headlen == 0 || head.body_len < 0 || head.body_len as u32 != body_len + || attr < 2 || head_len.checked_add(blen).and_then(|x| x.checked_add(attr)) != Some(existing.len()) { write_err(err, err_cap, "kvspace: write-in-place body_len mismatch"); return 1; } - hd.read_cache.remove(&key); + hd.cache_invalidate(&target_key); hd.write_buf = existing; - hd.pending_key = Some(key); + hd.pending_key = Some(target_key); unsafe { *body = hd.write_buf.as_mut_ptr().add(head_len) }; 0 } @@ -739,7 +813,7 @@ pub extern "C" fn kvspaceWriteNewPlace( }; /* 固定头：body 偏移恒 = PREFIX（=8）。 */ let head_len = crate::headlenpow::PREFIX; - hd.read_cache.remove(&key); + hd.cache_invalidate(&key); hd.write_buf = xv; hd.pending_key = Some(key); unsafe { *body = hd.write_buf.as_mut_ptr().add(head_len) }; From 44ed9732c36419c77dfe2293d80d52408d879d6f Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 18:22:42 +0000 Subject: [PATCH 09/12] tests: isolate Redis tests without FLUSHDB and add v0.3.23 ABI regression suite - cp.rs, map_coord.rs, common/mod.rs: replace kv.clear() with per-process prefixes + del_tree cleanup and skip cleanly when Redis is unreachable - add tests/review_fixes.rs covering forward/multi-hop/cyclic/>16-hop Ptr chains, WriteInPlace(resolve=1), GetHead/GetPart/SetPart resolution, untyped ST_SLACK round-trip + Watch, and attr trailer bounds - silence compiler warnings across test targets --- tests/common/mod.rs | 30 +++- tests/cp.rs | 124 +++++++++----- tests/map_coord.rs | 68 +++++--- tests/read_cache.rs | 12 +- tests/review_fixes.rs | 386 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 549 insertions(+), 71 deletions(-) create mode 100644 tests/review_fixes.rs diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 337b387..e4a62bf 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,7 +1,32 @@ // tests/common/mod.rs — 各集成测试共用的小工具。 // -// 只放「不需要真后端就能定义」的东西。语义测试用的 run(dsn) 仍然各自留在 -// cp.rs / map_coord.rs 里，保持每个测试文件自足。 +// Backend-independent helpers shared across integration tests; per-suite `run(dsn, root)` +// entry points stay in their respective test files (`cp.rs`, `map_coord.rs`, etc.). + +/// Returns the Redis test DSN if reachable, or logs and returns `None` to skip. +/// Tests must never call `clear()` (`FLUSHDB`); always isolate under a unique prefix + `del_tree`. +#[allow(dead_code)] +pub fn redis_test_dsn(label: &str) -> Option { + let dsn = std::env::var("KVSPACE_TEST_REDIS_DSN") + .unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string()); + let host_port = dsn + .trim_start_matches("redis://") + .split('/') + .next() + .unwrap_or(""); + let addr = host_port.rsplit('@').next().unwrap_or(host_port); + let reachable = std::net::ToSocketAddrs::to_socket_addrs(&addr) + .ok() + .and_then(|mut a| a.next()) + .is_some_and(|a| { + std::net::TcpStream::connect_timeout(&a, std::time::Duration::from_millis(300)).is_ok() + }); + if !reachable { + eprintln!("skipping redis {label} test: {addr} unreachable"); + return None; + } + Some(dsn) +} /// S3 集成测试的 DSN 与安全守卫。 /// @@ -14,6 +39,7 @@ /// KVSPACE='s3://AK:SK@bucket/_test/run1/?endpoint=…®ion=…' \ /// cargo test --test cp -- --ignored --nocapture /// ``` +#[allow(dead_code)] pub fn s3_test_dsn() -> Option { let dsn = match std::env::var("KVSPACE") { Ok(v) if !v.is_empty() => v, diff --git a/tests/cp.rs b/tests/cp.rs index 824b8cb..e540710 100644 --- a/tests/cp.rs +++ b/tests/cp.rs @@ -22,56 +22,88 @@ fn get_one(kv: &mut dyn KVSpace, key: &str) -> XValue { kv.get(&p, &[l], true).remove(0) } -fn run(dsn: &str) { +fn run(dsn: &str, root: &str) { let mut kv = conn(dsn); let kv: &mut dyn KVSpace = kv.as_mut(); - kv.clear().unwrap(); + kv.del_tree(root).unwrap(); + kv.mkindex(&format!("{root}/"), 0).unwrap(); // 原型 stringkeymap 容器：两个叶成员 + 一个嵌套容器成员。 - set(kv, "/proto", &new_map_langtype("[]char/utf32·int64")); - set(kv, "/proto·x", &new_int64(&[10])); - set(kv, "/proto·y", &new_int64(&[20])); - set(kv, "/proto·sub", &new_map_langtype("[]char/utf32·int64")); - set(kv, "/proto·sub·z", &new_int64(&[30])); - - kv.cp_tree("/proto", "/proto").unwrap(); - assert!(kv.cp_tree("/proto", "/proto/child").is_err()); - assert_eq!(get_one(kv, "/proto·x"), new_int64(&[10])); - - // 只读扩展源 + 原型上的 extindex 成员 ov 覆盖 /ext·（extindex 节点落 `/` 目录键 /proto·ov/）。 - set(kv, "/ext", &new_map_langtype("[]char/utf32·int64")); set( kv, - "/ext·", + &format!("{root}/proto"), + &new_map_langtype("[]char/utf32·int64"), + ); + set(kv, &format!("{root}/proto·x"), &new_int64(&[10])); + set(kv, &format!("{root}/proto·y"), &new_int64(&[20])); + set( + kv, + &format!("{root}/proto·sub"), + &new_map_langtype("[]char/utf32·int64"), + ); + set(kv, &format!("{root}/proto·sub·z"), &new_int64(&[30])); + + kv.cp_tree(&format!("{root}/proto"), &format!("{root}/proto")) + .unwrap(); + assert!(kv + .cp_tree(&format!("{root}/proto"), &format!("{root}/proto/child")) + .is_err()); + assert_eq!(get_one(kv, &format!("{root}/proto·x")), new_int64(&[10])); + + // Read-only ext source + extindex member `ov` overlaying `{root}/ext·` at `{root}/proto·ov/`. + set( + kv, + &format!("{root}/ext"), + &new_map_langtype("[]char/utf32·int64"), + ); + let ext_loc = format!("{root}/ext·"); + set( + kv, + &ext_loc, &XValue::Opaque(Opaque { kind: "lib".into(), body: vec![], array_len: 1, }), ); - set(kv, "/ext·a", &new_int64(&[99])); - kv.ext_index("/proto·ov/", "/ext·").unwrap(); + set(kv, &format!("{root}/ext·a"), &new_int64(&[99])); + kv.ext_index(&format!("{root}/proto·ov/"), &ext_loc) + .unwrap(); // ── cp：单 key 拷贝，不带成员 ── - kv.cp("/proto·x", "/px").unwrap(); - assert_eq!(get_one(kv, "/px"), new_int64(&[10]), "cp 单 key"); + kv.cp(&format!("{root}/proto·x"), &format!("{root}/px")) + .unwrap(); + assert_eq!( + get_one(kv, &format!("{root}/px")), + new_int64(&[10]), + "cp single key" + ); // ── cp_tree：递归子树拷贝 ── - kv.cp_tree("/proto", "/inst").unwrap(); - assert_eq!(get_one(kv, "/inst·x"), new_int64(&[10]), "cp_tree 叶成员 x"); - assert_eq!(get_one(kv, "/inst·y"), new_int64(&[20]), "cp_tree 叶成员 y"); + kv.cp_tree(&format!("{root}/proto"), &format!("{root}/inst")) + .unwrap(); assert_eq!( - get_one(kv, "/inst·sub·z"), + get_one(kv, &format!("{root}/inst·x")), + new_int64(&[10]), + "cp_tree leaf member x" + ); + assert_eq!( + get_one(kv, &format!("{root}/inst·y")), + new_int64(&[20]), + "cp_tree leaf member y" + ); + assert_eq!( + get_one(kv, &format!("{root}/inst·sub·z")), new_int64(&[30]), "cp_tree 嵌套成员 z" ); - match get_one(kv, "/inst") { + match get_one(kv, &format!("{root}/inst")) { XValue::Map(_) => {} other => panic!("cp_tree 根值 kind 丢失: {:?}", other), } // 成员表完整（含嵌套与 extindex 成员）。 - let mut m = kv.list("/inst·", false, true); + let mut m = kv.list(&format!("{root}/inst·"), false, true); m.sort(); assert!( m.contains(&"x".to_string()) @@ -83,48 +115,62 @@ fn run(dsn: &str) { // 根登记进父 index。 assert!( - kv.list("/", false, true).contains(&"inst".to_string()), + kv.list(&format!("{root}/"), false, true) + .contains(&"inst".to_string()), "cp_tree 根登记父 index" ); // ── 关键：extindex 成员在 target 侧成为新 extindex，overlay 读通 ── assert_eq!( - kv.get("/inst·ov/", &["a".into()], true).remove(0), + kv.get(&format!("{root}/inst·ov/"), &["a".into()], true) + .remove(0), new_int64(&[99]), "cp_tree 的 extindex 成员 overlay 读通" ); - match kv.get("/inst·", &["ov/".into()], true).remove(0) { - XValue::Ext(e) => assert_eq!((e.langtype.as_str(), e.locator.as_str()), ("lib", "/ext·")), + match kv + .get(&format!("{root}/inst·"), &["ov/".into()], true) + .remove(0) + { + XValue::Ext(e) => assert_eq!( + (e.langtype.as_str(), e.locator.as_str()), + ("lib", ext_loc.as_str()) + ), other => panic!("target 侧应为 extindex: {:?}", other), } - kv.cp_list("/proto", "/shallow").unwrap(); + kv.cp_list(&format!("{root}/proto"), &format!("{root}/shallow")) + .unwrap(); assert_eq!( - kv.get("/shallow·ov/", &["a".into()], true).remove(0), + kv.get(&format!("{root}/shallow·ov/"), &["a".into()], true) + .remove(0), new_int64(&[99]) ); // 改动源 extindex 目标，target 因共享同一只读扩展而同步可见（非深拷贝底层数据）。 - set(kv, "/ext·a", &new_int64(&[123])); + set(kv, &format!("{root}/ext·a"), &new_int64(&[123])); assert_eq!( - kv.get("/inst·ov/", &["a".into()], true).remove(0), + kv.get(&format!("{root}/inst·ov/"), &["a".into()], true) + .remove(0), new_int64(&[123]), "extindex 共享同一只读扩展" ); + + kv.del_tree(root).unwrap(); } #[test] fn cp_redis() { - let dsn = std::env::var("KVSPACE_TEST_REDIS_DSN") - .unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string()); - run(&dsn); + let Some(dsn) = common::redis_test_dsn("cp") else { + return; + }; + run(&dsn, &format!("/kvspace-cp-test-{}", std::process::id())); } #[test] fn cp_fs() { - let dir = std::env::temp_dir().join("kvspace-cp-test"); + let dir = std::env::temp_dir().join(format!("kvspace-cp-test-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); - run(&format!("fs://{}", dir.display())); + run(&format!("fs://{}", dir.display()), "/r"); let _ = std::fs::remove_dir_all(&dir); } @@ -134,5 +180,5 @@ fn cp_fs() { #[ignore = "需要真桶凭据，cargo test -- --ignored 才跑"] fn cp_s3() { let Some(dsn) = common::s3_test_dsn() else { return }; - run(&dsn); + run(&dsn, &format!("/kvspace-cp-test-{}", std::process::id())); } diff --git a/tests/map_coord.rs b/tests/map_coord.rs index a0c4f82..7c52716 100644 --- a/tests/map_coord.rs +++ b/tests/map_coord.rs @@ -22,63 +22,82 @@ fn get_one(kv: &mut dyn KVSpace, key: &str) -> XValue { kv.get(&p, &[l], true).remove(0) } -fn run(dsn: &str) { +fn run(dsn: &str, root: &str) { let mut kv = conn(dsn); let kv: &mut dyn KVSpace = kv.as_mut(); - kv.clear().unwrap(); + kv.del_tree(root).unwrap(); + kv.mkindex(&format!("{root}/"), 0).unwrap(); - set(kv, "/m", &new_map_langtype("[int64,int64]·float32")); + set( + kv, + &format!("{root}/m"), + &new_map_langtype("[int64,int64]·float32"), + ); // 乱序写坐标成员。 - set(kv, "/m·[1,2]", &new_float32(&[6.28])); - set(kv, "/m·[0,1]", &new_float32(&[3.14])); - set(kv, "/m·[0,0]", &new_float32(&[1.0])); + set(kv, &format!("{root}/m·[1,2]"), &new_float32(&[6.28])); + set(kv, &format!("{root}/m·[0,1]"), &new_float32(&[3.14])); + set(kv, &format!("{root}/m·[0,0]"), &new_float32(&[1.0])); - // The map type is stored at /m; members are physical /m· keys. - match get_one(kv, "/m") { + // The map type is stored at {root}/m; members are physical {root}/m· keys. + match get_one(kv, &format!("{root}/m")) { XValue::Map(m) => assert_eq!(m.langtype, "[int64,int64]·float32"), other => panic!("容器值 kind 丢失: {:?}", other), } // list 按 row-major 数值升序（先比 s0 再比 s1）。 - let names = kv.list("/m·", false, true); + let names = kv.list(&format!("{root}/m·"), false, true); assert_eq!(names, vec!["[0,0]", "[0,1]", "[1,2]"], "list 顺序"); // 坐标成员读回，缺席坐标读 None。 - assert_eq!(get_one(kv, "/m·[1,2]"), new_float32(&[6.28])); - assert!(is_none(&get_one(kv, "/m·[9,9]"))); + assert_eq!(get_one(kv, &format!("{root}/m·[1,2]")), new_float32(&[6.28])); + assert!(is_none(&get_one(kv, &format!("{root}/m·[9,9]")))); // 未声明容器直接写坐标成员 → **拒绝**：memhead 不存在则禁止写 memitem。 // 这条规则两侧同源——kvspace（本处）与 runtime 的 kvlangBuiltinCheckMemhead； // 曾经只有 redis 后端会兜底自动建容器，fs 不会，同一份代码两后端分叉，故铲掉兜底。 let e = kv.set(&[KVPair { - key: "/n·[2,3]".to_string(), + key: format!("{root}/n·[2,3]"), val: new_int64(&[7]), raw: None, }]); assert!(e.is_err(), "未声明 memhead 的成员写必须被拒: {e:?}"); - assert!(is_none(&get_one(kv, "/n")), "被拒后不应留下容器值"); + assert!( + is_none(&get_one(kv, &format!("{root}/n"))), + "rejected member write must not leave container value behind" + ); // 命名成员仍为裸名，与坐标段字面可分。 - set(kv, "/h", &new_map_langtype("[]char/utf32·int64")); - set(kv, "/h·x", &new_int64(&[1])); - set(kv, "/h·[0]", &new_int64(&[2])); - let names = kv.list("/h·", false, true); + set( + kv, + &format!("{root}/h"), + &new_map_langtype("[]char/utf32·int64"), + ); + set(kv, &format!("{root}/h·x"), &new_int64(&[1])); + set(kv, &format!("{root}/h·[0]"), &new_int64(&[2])); + let names = kv.list(&format!("{root}/h·"), false, true); assert!(names.contains(&"x".to_string()) && names.contains(&"[0]".to_string())); + + kv.del_tree(root).unwrap(); } #[test] fn map_coord_redis() { - let dsn = std::env::var("KVSPACE_TEST_REDIS_DSN") - .unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string()); - run(&dsn); + let Some(dsn) = common::redis_test_dsn("map_coord") else { + return; + }; + run( + &dsn, + &format!("/kvspace-map-coord-test-{}", std::process::id()), + ); } #[test] fn map_coord_fs() { - let dir = std::env::temp_dir().join("kvspace-map-coord-test"); + let dir = + std::env::temp_dir().join(format!("kvspace-map-coord-test-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); - run(&format!("fs://{}", dir.display())); + run(&format!("fs://{}", dir.display()), "/r"); let _ = std::fs::remove_dir_all(&dir); } @@ -88,5 +107,8 @@ fn map_coord_fs() { #[ignore = "需要真桶凭据，cargo test -- --ignored 才跑"] fn map_coord_s3() { let Some(dsn) = common::s3_test_dsn() else { return }; - run(&dsn); + run( + &dsn, + &format!("/kvspace-map-coord-test-{}", std::process::id()), + ); } diff --git a/tests/read_cache.rs b/tests/read_cache.rs index ec4b4d9..f38b016 100644 --- a/tests/read_cache.rs +++ b/tests/read_cache.rs @@ -16,7 +16,7 @@ fn attr_of(lt: &str) -> Vec { return vec![2, 0]; } let mut v = Vec::new(); - let alen = (2u16 + 3 + lt.len() as u16); + let alen = 2u16 + 3 + lt.len() as u16; v.extend_from_slice(&alen.to_le_bytes()); v.push(1); // tag = langtype v.extend_from_slice(&(lt.len() as u16).to_le_bytes()); @@ -334,8 +334,8 @@ fn volatile_status_observed_and_double_buffered() { let dsn = dsn(); let mut c = dsn.as_bytes().to_vec(); c.push(0); - let a = unsafe { kvspaceConnect(c.as_ptr() as *const _) }; - let b = unsafe { kvspaceConnect(c.as_ptr() as *const _) }; + let a = kvspaceConnect(c.as_ptr() as *const _); + let b = kvspaceConnect(c.as_ptr() as *const _); assert!(!a.is_null() && !b.is_null()); let st = "/vthread/1\u{2025}status"; unsafe { @@ -380,9 +380,7 @@ fn volatile_status_observed_and_double_buffered() { "snapshot keys ignore external writes" ); } - unsafe { - kvspaceClose(a); - kvspaceClose(b); - } + kvspaceClose(a); + kvspaceClose(b); std::fs::remove_dir_all(dsn.trim_start_matches("fs://")).ok(); } diff --git a/tests/review_fixes.rs b/tests/review_fixes.rs new file mode 100644 index 0000000..6162142 --- /dev/null +++ b/tests/review_fixes.rs @@ -0,0 +1,386 @@ +// review_fixes.rs - Cross-backend C ABI compatibility regression tests (v0.3.23). +// +// Covers: +// 1. Unchecked Ptr storage in SetValue (dangling/forward Ptr, 2-hop slot -> ‥arg -> caller, +// char/utf8 -> []char/utf32 parameter binding, pointer cycles, and `[1]·int64` map langtype). +// 2. kvspaceGet(resolve=0 vs 1) leaf Ptr chain resolution (up to 16 hops, coexisting raw and +// resolved borrowed views, missing target -> 0 + NULL, cycle / >16 hops -> 1). +// 3. kvspaceGetHead / kvspaceGetPart / kvspaceSetPart leaf Ptr resolution and cache invalidation. +// 4. kvspaceWriteInPlace(resolve=1) mutating the terminal target through a Ptr chain. +// 5. Untyped variable-length system slots (ST_SLACK + empty langtype) round-trip and Watch. +// 6. SetValue attr trailer validation and trailing padding trim. +// 7. /lib//‥ctime cross-handle hot-reload invalidation with retired view retention. +// 8. List(resolve=1) following Ptr links whose targets lack a trailing `/` or `·`. + +use kvspace_durable::ffi::*; +use kvspace_durable::*; +use std::ffi::{c_char, CStr, CString}; + +mod common; + +unsafe fn connect_c(dsn: &str) -> *mut Handle { + let c = CString::new(dsn).unwrap(); + let h = kvspaceConnect(c.as_ptr()); + assert!(!h.is_null(), "connect failed: {dsn}"); + h +} + +unsafe fn set_raw_checked(h: *mut Handle, key: &str, wire: &[u8]) { + let k = CString::new(key).unwrap(); + let mut err = [0 as c_char; 256]; + let rc = kvspaceSetValue( + h, + k.as_ptr(), + wire.as_ptr(), + wire.len() as u32, + err.as_mut_ptr(), + err.len() as u32, + ); + let msg = CStr::from_ptr(err.as_ptr()).to_string_lossy(); + assert_eq!(rc, 0, "SetValue({key}) failed: {msg}"); +} + +unsafe fn get_raw_c(h: *mut Handle, key: &str, resolve: i32) -> (i32, *mut u8, u32) { + let k = CString::new(key).unwrap(); + let mut out: *mut u8 = std::ptr::null_mut(); + let mut len: u32 = 0; + let rc = kvspaceGet(h, k.as_ptr(), resolve, &mut out, &mut len); + (rc, out, len) +} + +unsafe fn deltree_c(h: *mut Handle, prefix: &str) { + let p = CString::new(prefix).unwrap(); + let mut err = [0 as c_char; 256]; + let rc = kvspaceDelTree(h, p.as_ptr(), err.as_mut_ptr(), err.len() as u32); + let msg = CStr::from_ptr(err.as_ptr()).to_string_lossy(); + assert_eq!(rc, 0, "DelTree({prefix}) failed: {msg}"); +} + +unsafe fn run_backend_suite(dsn: &str, root: &str) { + let h = connect_c(dsn); + deltree_c(h, root); + + // 1. Forward/dangling Ptr write + resolve=0 vs resolve=1. + let k_dangling = format!("{root}/dangling_ptr"); + let k_late_target = format!("{root}/late_target"); + let w_dangling = new_ptr("int64", &k_late_target).to_wire(); + set_raw_checked(h, &k_dangling, &w_dangling); + + // Before target exists: resolve=0 returns raw Ptr wire; resolve=1 returns 0 with NULL/0. + let (rc0, p0, l0) = get_raw_c(h, &k_dangling, 0); + assert_eq!(rc0, 0); + assert!(!p0.is_null() && l0 as usize == w_dangling.len()); + assert_eq!( + std::slice::from_raw_parts(p0, l0 as usize), + w_dangling.as_slice() + ); + + let (rc1, p1, l1) = get_raw_c(h, &k_dangling, 1); + assert_eq!(rc1, 0, "dangling Ptr with resolve=1 must return 0 like a missing key"); + assert!(p1.is_null() && l1 == 0, "dangling Ptr with resolve=1 must clear output"); + + // Once the forward target is written, resolve=1 resolves to the new value. + let w_val42 = new_int64(&[42]).to_wire(); + set_raw_checked(h, &k_late_target, &w_val42); + let (rc1b, p1b, l1b) = get_raw_c(h, &k_dangling, 1); + assert_eq!(rc1b, 0); + assert!(!p1b.is_null() && l1b as usize == w_val42.len()); + assert_eq!( + std::slice::from_raw_parts(p1b, l1b as usize), + w_val42.as_slice() + ); + + // 2. 2-hop Ptr (slot -> ‥arg -> caller), heterogeneous kindexpr, and coexisting borrowed views. + let k_caller = format!("{root}/caller_slot"); + let k_arg = format!("{root}/\u{2025}arg[0,-1]"); + let k_slot = format!("{root}/[1,-1]"); + let w_utf8 = new_char_byte("你好".as_bytes()).to_wire(); + // Actual value is char/utf8 while parameter kindexpr is []char/utf32 across 2 hops. + set_raw_checked(h, &k_caller, &w_utf8); + let w_arg_ptr = new_ptr("[]char/utf32", &k_caller).to_wire(); + let w_slot_ptr = new_ptr("[]char/utf32", &k_arg).to_wire(); + set_raw_checked(h, &k_arg, &w_arg_ptr); + set_raw_checked(h, &k_slot, &w_slot_ptr); + + let (rc_raw, ptr_raw, len_raw) = get_raw_c(h, &k_slot, 0); + let (rc_res, ptr_res, len_res) = get_raw_c(h, &k_slot, 1); + assert_eq!((rc_raw, rc_res), (0, 0)); + assert!(!ptr_raw.is_null() && !ptr_res.is_null()); + // Raw Ptr view and resolved target view stay valid simultaneously in read_cache. + assert_eq!( + std::slice::from_raw_parts(ptr_raw, len_raw as usize), + w_slot_ptr.as_slice() + ); + assert_eq!( + std::slice::from_raw_parts(ptr_res, len_res as usize), + w_utf8.as_slice() + ); + + // kvspaceGetHead follows the 2-hop Ptr chain to read the terminal target header/langtype. + let k_slot_c = CString::new(k_slot.as_str()).unwrap(); + let mut head_out: kvspaceHead_t = std::mem::zeroed(); + assert_eq!(kvspaceGetHead(h, k_slot_c.as_ptr(), &mut head_out), 0); + assert_eq!(head_out.r#ref, REF_INLINE); + assert_eq!(head_out.storetype, ST_SLACK); + assert_eq!(head_out.body_len, 6); + let lt_str = std::str::from_utf8(&head_out.langtype[..head_out.langtype_len as usize]).unwrap(); + assert_eq!(lt_str, "[6]char/utf8"); + + // 3. 16-hop limit and cycle detection (>16 hops or cycle returns 1). + let k_leaf16 = format!("{root}/hop_leaf"); + let w_leaf99 = new_int64(&[99]).to_wire(); + set_raw_checked(h, &k_leaf16, &w_leaf99); + // hop_1 -> hop_leaf (1 hop), ..., hop_16 -> hop_15 (16 hops, valid); hop_17 -> hop_16 (17 hops, error). + let mut prev = k_leaf16.clone(); + for i in 1..=17 { + let cur = format!("{root}/hop_{i}"); + let wp = new_ptr("int64", &prev).to_wire(); + set_raw_checked(h, &cur, &wp); + prev = cur; + } + let (rc_16, p_16, l_16) = get_raw_c(h, &format!("{root}/hop_16"), 1); + assert_eq!(rc_16, 0, "16-hop Ptr chain must succeed"); + assert_eq!( + std::slice::from_raw_parts(p_16, l_16 as usize), + w_leaf99.as_slice() + ); + + let (rc_17, p_17, l_17) = get_raw_c(h, &format!("{root}/hop_17"), 1); + assert_eq!(rc_17, 1, "17-hop Ptr chain (>16) must return 1"); + assert!(p_17.is_null() && l_17 == 0); + + // 2-node cycle: SetValue accepts cyclic pointers; resolve=0 reads raw Ptr, resolve=1 returns 1. + let k_cyc_a = format!("{root}/cyc_a"); + let k_cyc_b = format!("{root}/cyc_b"); + set_raw_checked(h, &k_cyc_a, &new_ptr("int64", &k_cyc_b).to_wire()); + set_raw_checked(h, &k_cyc_b, &new_ptr("int64", &k_cyc_a).to_wire()); + let (rc_cyc0, p_cyc0, _) = get_raw_c(h, &k_cyc_a, 0); + assert_eq!(rc_cyc0, 0); + assert!(!p_cyc0.is_null()); + let (rc_cyc1, p_cyc1, l_cyc1) = get_raw_c(h, &k_cyc_a, 1); + assert_eq!(rc_cyc1, 1, "Ptr cycle with resolve=1 must return 1"); + assert!(p_cyc1.is_null() && l_cyc1 == 0); + + // 4. kvspaceWriteInPlace(resolve=1) mutates terminal target through a 2-hop Ptr. + let k_wip_val = format!("{root}/wip_val"); + let k_wip_p1 = format!("{root}/wip_p1"); + let k_wip_p2 = format!("{root}/wip_p2"); + let w_init = new_int64(&[100]).to_wire(); + let w_p1 = new_ptr("int64", &k_wip_val).to_wire(); + let w_p2 = new_ptr("int64", &k_wip_p1).to_wire(); + set_raw_checked(h, &k_wip_val, &w_init); + set_raw_checked(h, &k_wip_p1, &w_p1); + set_raw_checked(h, &k_wip_p2, &w_p2); + + // Prime read_cache with wip_val first to verify WriteInPlace(wip_p2, resolve=1) invalidates it. + let (rc_pre, p_pre, l_pre) = get_raw_c(h, &k_wip_val, 0); + assert_eq!(rc_pre, 0); + assert_eq!( + std::slice::from_raw_parts(p_pre, l_pre as usize), + w_init.as_slice() + ); + + let k_wip_p2_c = CString::new(k_wip_p2.as_str()).unwrap(); + let mut body_ptr: *mut u8 = std::ptr::null_mut(); + let mut err = [0 as c_char; 256]; + let rc_wip = kvspaceWriteInPlace( + h, + k_wip_p2_c.as_ptr(), + 1, + 8, + &mut body_ptr, + err.as_mut_ptr(), + err.len() as u32, + ); + assert_eq!(rc_wip, 0, "WriteInPlace(resolve=1) through 2-hop Ptr failed"); + assert!(!body_ptr.is_null()); + std::ptr::copy_nonoverlapping(777i64.to_le_bytes().as_ptr(), body_ptr, 8); + + // Next read flushes: terminal target becomes 777 while wip_p2 / wip_p1 keep their Ptr wire. + let w_777 = new_int64(&[777]).to_wire(); + let (rc_post_res, p_post_res, l_post_res) = get_raw_c(h, &k_wip_p2, 1); + assert_eq!(rc_post_res, 0); + assert_eq!( + std::slice::from_raw_parts(p_post_res, l_post_res as usize), + w_777.as_slice() + ); + let (_, p_post_val, l_post_val) = get_raw_c(h, &k_wip_val, 0); + assert_eq!( + std::slice::from_raw_parts(p_post_val, l_post_val as usize), + w_777.as_slice() + ); + let (_, p_post_p2, l_post_p2) = get_raw_c(h, &k_wip_p2, 0); + assert_eq!( + std::slice::from_raw_parts(p_post_p2, l_post_p2 as usize), + w_p2.as_slice() + ); + + // 5. GetPart / SetPart follow Ptr chains. + let mut part_out: *mut u8 = std::ptr::null_mut(); + let mut part_len: u32 = 0; + assert_eq!( + kvspaceGetPart(h, k_wip_p2_c.as_ptr(), 8, 8, &mut part_out, &mut part_len), + 0 + ); + assert_eq!(part_len, 8); + assert_eq!( + std::slice::from_raw_parts(part_out, 8), + &777i64.to_le_bytes() + ); + + let new_body = 888i64.to_le_bytes(); + assert_eq!( + kvspaceSetPart( + h, + k_wip_p2_c.as_ptr(), + 8, + new_body.as_ptr(), + 8, + err.as_mut_ptr(), + err.len() as u32 + ), + 0 + ); + let (_, p_after_sp, l_after_sp) = get_raw_c(h, &k_wip_val, 0); + assert_eq!( + std::slice::from_raw_parts(p_after_sp, l_after_sp as usize), + new_int64(&[888]).to_wire().as_slice() + ); + + // 6. Untyped variable-length system slots (ST_SLACK + empty langtype) round-trip and Watch. + let k_sys_lib = format!("{root}/vthread/1/[1]/\u{2025}lib"); + let sys_body = b"/lib/main\xc2\xb7test"; + let w_sys_lib = headlenpow::to_wire(ST_SLACK, REF_INLINE, sys_body, "").unwrap(); + set_raw_checked(h, &k_sys_lib, &w_sys_lib); + + let (rc_sys, p_sys, l_sys) = get_raw_c(h, &k_sys_lib, 0); + assert_eq!(rc_sys, 0); + let got_sys = std::slice::from_raw_parts(p_sys, l_sys as usize); + assert_eq!(got_sys, w_sys_lib.as_slice(), "untyped ST_SLACK must round-trip verbatim"); + + // from_wire must not treat untyped ST_SLACK as XValue::None or drop its body. + let decoded_sys = from_wire(got_sys); + assert!(!is_none(&decoded_sys), "ST_SLACK with empty langtype must not decode as None"); + assert_eq!(decoded_sys.to_wire(), w_sys_lib); + + // Canonical None (ST_SHORT + REF_INLINE + len=0 + empty langtype) still decodes to XValue::None. + let w_none = headlenpow::to_wire(ST_SHORT, REF_INLINE, &[], "").unwrap(); + assert!(is_none(&from_wire(&w_none))); + + // Empty struct marker (ST_SHORT + len=0 + langtype="/lib/Point") preserves its wire bytes. + let k_struct = format!("{root}/pt"); + let w_struct = headlenpow::to_wire(ST_SHORT, REF_INLINE, &[], "/lib/Point").unwrap(); + set_raw_checked(h, &k_struct, &w_struct); + set_raw_checked(h, &format!("{k_struct}\u{b7}x"), &new_int64(&[3]).to_wire()); + let (_, p_st, l_st) = get_raw_c(h, &k_struct, 0); + assert_eq!( + std::slice::from_raw_parts(p_st, l_st as usize), + w_struct.as_slice() + ); + + // Watch on an untyped status slot: blocks while "running", returns full ST_SLACK wire on "stopped". + let k_status = format!("{root}/status_slot"); + let w_running = headlenpow::to_wire(ST_SLACK, REF_INLINE, b"running", "").unwrap(); + let w_stopped = headlenpow::to_wire(ST_SLACK, REF_INLINE, b"stopped", "").unwrap(); + set_raw_checked(h, &k_status, &w_running); + + let dsn_bg = dsn.to_string(); + let k_status_bg = k_status.clone(); + let w_stopped_bg = w_stopped.clone(); + let writer = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(25)); + unsafe { + let h2 = connect_c(&dsn_bg); + set_raw_checked(h2, &k_status_bg, &w_stopped_bg); + kvspaceClose(h2); + } + }); + + let k_status_c = CString::new(k_status.as_str()).unwrap(); + let mut watch_out: *mut u8 = std::ptr::null_mut(); + let mut watch_len: u32 = 0; + let rc_w = kvspaceWatch( + h, + k_status_c.as_ptr(), + w_stopped.as_ptr(), + w_stopped.len() as u32, + 5_000_000, // 5ms tick + &mut watch_out, + &mut watch_len, + ); + writer.join().unwrap(); + assert_eq!(rc_w, 0); + assert_eq!( + std::slice::from_raw_parts(watch_out, watch_len as usize), + w_stopped.as_slice(), + "kvspaceWatch on untyped ST_SLACK must preserve body and ST_SLACK header" + ); + + // 7. SetValue attr trailer validation (matches kvspace-c). + let k_bad = CString::new(format!("{root}/bad_attr")).unwrap(); + let noattr: [u8; 16] = [8, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0]; + assert_ne!( + kvspaceSetValue(h, k_bad.as_ptr(), noattr.as_ptr(), noattr.len() as u32, std::ptr::null_mut(), 0), + 0, + "wire value without attr trailer must be rejected" + ); + let shortattr: [u8; 18] = [8, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 1, 0]; + assert_ne!( + kvspaceSetValue(h, k_bad.as_ptr(), shortattr.as_ptr(), shortattr.len() as u32, std::ptr::null_mut(), 0), + 0, + "attr_len < 2 must be rejected" + ); + let padded: [u8; 20] = [ + 8, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0xEE, 0xEE, + ]; + let k_pad = format!("{root}/padded"); + set_raw_checked(h, &k_pad, &padded); + let (_, p_pad, l_pad) = get_raw_c(h, &k_pad, 0); + assert_eq!(l_pad, 18, "trailing padding bytes must be trimmed to 8 + len + attr"); + assert_eq!(std::slice::from_raw_parts(p_pad, l_pad as usize), &padded[..18]); + + // 8. Map langtype normalization in validate_ptr (preserve `[1]·int64` key type). + { + let mut kv = conn(dsn); + let k_map = format!("{root}/num_key_map"); + kv.set(&[KVPair { + key: k_map.clone(), + val: new_map_langtype("[1]·int64"), + raw: None, + }]) + .unwrap(); + assert!( + validate_ptr(kv.as_mut(), &k_map, "[1]·int64").is_ok(), + "map langtype [1]·int64 must match itself" + ); + assert!( + validate_ptr(kv.as_mut(), &k_map, "[2]·int64").is_err(), + "distinct map key shapes [1]·int64 and [2]·int64 must not be stripped by norm_langtype" + ); + } + + deltree_c(h, root); + kvspaceClose(h); +} + +#[test] +fn review_fixes_fs() { + let dir = std::env::temp_dir().join(format!("kvspace-review-fixes-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let dsn = format!("fs://{}", dir.display()); + unsafe { + run_backend_suite(&dsn, "/rf"); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn review_fixes_redis() { + let Some(dsn) = common::redis_test_dsn("review_fixes") else { + return; + }; + let root = format!("/kvspace-review-fixes-test-{}", std::process::id()); + unsafe { + run_backend_suite(&dsn, &root); + } +} From bfc80598234abfb03e044789948e103a222f5065 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 19:14:18 +0000 Subject: [PATCH 10/12] ffi: retire cached function entries when /lib/*/ctime changes across handles Detect function ctime control-key updates in volatile_refresh and move cached snapshots under that function directory into retired so cross-handle relayout updates take effect immediately while keeping borrowed pointers valid until Close. --- src/ffi.rs | 48 ++++++++++++++++++++++++++++++++++--------- tests/review_fixes.rs | 46 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index e100f35..10ae29c 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -40,7 +40,8 @@ pub struct VolatileBuf { /// 外部可改写、必须每读重读后端的控制键（normative：见 spec CABI）。 fn is_volatile(key: &str) -> bool { - key.starts_with("/vthread/") && key.ends_with("\u{2025}status") + (key.starts_with("/vthread/") && key.ends_with("\u{2025}status")) + || (key.starts_with("/lib/") && key.ends_with("/\u{2025}ctime")) } pub struct Handle { @@ -173,20 +174,47 @@ fn under_prefix(key: &str, prefix: &str) -> bool { } /// 控制键重读后端并写入双缓冲的另一块（上一块保活到该 key 下一次读）。 +/// When `/lib//‥ctime` changes, moves cached entries under `` into `retired` +/// so cross-handle relayout updates are visible immediately without invalidating borrowed pointers. fn volatile_refresh(hd: &mut Handle, key: &str) -> bool { let raw = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hd.kv.get_raw(key))) { Ok(r) => r, Err(_) => return false, }; - let vb = hd - .volatile_cache - .entry(key.to_string()) - .or_insert_with(|| VolatileBuf { - slots: [Vec::new(), Vec::new()], - which: 0, - }); - vb.which ^= 1; - vb.slots[vb.which] = raw; + let mut ctime_changed_dir: Option<&str> = None; + { + let vb = hd + .volatile_cache + .entry(key.to_string()) + .or_insert_with(|| VolatileBuf { + slots: [Vec::new(), Vec::new()], + which: 0, + }); + let prev = &vb.slots[vb.which]; + if !prev.is_empty() && prev.as_slice() != raw.as_slice() { + if let Some(func_dir) = key.strip_suffix("/\u{2025}ctime") { + if func_dir.starts_with("/lib/") && func_dir.len() > "/lib/".len() { + ctime_changed_dir = Some(func_dir); + } + } + } + vb.which ^= 1; + vb.slots[vb.which] = raw; + } + if let Some(func_dir) = ctime_changed_dir { + let stale_keys: Vec = hd + .read_cache + .keys() + .filter(|k| under_prefix(k, func_dir)) + .cloned() + .collect(); + for k in stale_keys { + if let Some(old_vec) = hd.read_cache.remove(&k) { + hd.retired.push(old_vec); + } + } + hd.list_cache = None; + } true } diff --git a/tests/review_fixes.rs b/tests/review_fixes.rs index 6162142..3e75f4c 100644 --- a/tests/review_fixes.rs +++ b/tests/review_fixes.rs @@ -359,6 +359,52 @@ unsafe fn run_backend_suite(dsn: &str, root: &str) { ); } + // 9. /lib//‥ctime cross-handle hot-reload detection and retired snapshot retention. + let func_dir = format!("/lib/rf_hot_{}", std::process::id()); + let k_ctime = format!("{func_dir}/\u{2025}ctime"); + let k_insn = format!("{func_dir}/[1,0]"); + let w_ctime1 = new_int64(&[1000]).to_wire(); + let w_insn1 = headlenpow::to_wire(ST_SLACK, REF_INLINE, b"ret 10", "").unwrap(); + set_raw_checked(h, &k_ctime, &w_ctime1); + set_raw_checked(h, &k_insn, &w_insn1); + + // Prime handle `h` with ‥ctime and [1,0], keeping a borrowed pointer to the old [1,0] view. + let (rc_ct1, p_ct1, l_ct1) = get_raw_c(h, &k_ctime, 0); + let (rc_in1, p_in1, l_in1) = get_raw_c(h, &k_insn, 0); + assert_eq!((rc_ct1, rc_in1), (0, 0)); + assert_eq!(std::slice::from_raw_parts(p_ct1, l_ct1 as usize), w_ctime1.as_slice()); + assert_eq!(std::slice::from_raw_parts(p_in1, l_in1 as usize), w_insn1.as_slice()); + + // Simulate external relayout via `h2` updating [1,0] and ‥ctime. + let h2 = connect_c(dsn); + let w_ctime2 = new_int64(&[2000]).to_wire(); + let w_insn2 = headlenpow::to_wire(ST_SLACK, REF_INLINE, b"ret 20", "").unwrap(); + set_raw_checked(h2, &k_insn, &w_insn2); + set_raw_checked(h2, &k_ctime, &w_ctime2); + kvspaceClose(h2); + + // Re-reading ‥ctime on `h` observes the new timestamp, retires cached [1,0], and keeps p_in1 valid. + let (rc_ct2, p_ct2, l_ct2) = get_raw_c(h, &k_ctime, 0); + assert_eq!(rc_ct2, 0); + assert_eq!( + std::slice::from_raw_parts(p_ct2, l_ct2 as usize), + w_ctime2.as_slice(), + "cross-handle /lib/*/‥ctime update must be visible immediately" + ); + let (rc_in2, p_in2, l_in2) = get_raw_c(h, &k_insn, 0); + assert_eq!(rc_in2, 0); + assert_eq!( + std::slice::from_raw_parts(p_in2, l_in2 as usize), + w_insn2.as_slice(), + "instructions under func_dir must reload after ‥ctime changes" + ); + assert_eq!( + std::slice::from_raw_parts(p_in1, l_in1 as usize), + w_insn1.as_slice(), + "retired instruction snapshot must remain valid until Close" + ); + deltree_c(h, &func_dir); + deltree_c(h, root); kvspaceClose(h); } From 6d8219abc35ff2373f40c12b27b7e481e83ebe35 Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 20:23:09 +0000 Subject: [PATCH 11/12] kvspace-durable: align cross-process volatile invalidation, fs dir_children, and bare-target Ptr listing - ffi: treat /lib/*/vids/* as volatile, retire read_cache across Watch and vids handoff Del, and update VolatileBuf - fs: list directory child without trailing slash only when __self__ exists without __dir__, and preserve trailing directory/member suffix when resolving links to bare targets - backend: preserve trailing directory/member suffix when resolving links to bare targets; add regression cases in tests/review_fixes.rs --- src/backend.rs | 30 ++++++++++++++++++++----- src/ffi.rs | 22 ++++++++++++++---- src/fs/kvspace.rs | 37 +++++++++++++++++++++++++----- src/kvspace_common.rs | 5 +++-- tests/review_fixes.rs | 52 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 17 deletions(-) diff --git a/src/backend.rs b/src/backend.rs index a661afa..37eb366 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -40,13 +40,14 @@ impl Backend { fn resolve_path(&self, path: &str) -> String { let mut path = path.to_string(); - loop { + for _ in 0..16 { let (resolved, changed) = self.resolve_one(&path); if !changed { return resolved; } path = resolved; } + path } fn resolve_parent(&self, path: &str) -> String { @@ -72,7 +73,19 @@ impl Backend { if path == PATH_SEP { return (path.to_string(), false); } - let trimmed = path.trim_matches('/'); + let dir_suf = if path.ends_with(OBJ_SEP) { + Some(OBJ_SEP) + } else if path.ends_with(DIR_INDEX_SUF) { + Some(DIR_INDEX_SUF) + } else { + None + }; + let clean = if dir_suf.is_some() { + strip_dir_suf(path) + } else { + path + }; + let trimmed = clean.trim_matches('/'); let parts: Vec<&str> = if trimmed.is_empty() { Vec::new() } else { @@ -93,10 +106,17 @@ impl Backend { let v = from_wire(data); if is_ptr(&v) { let target = ptr_target(&v); - if i + 1 < parts.len() { - return (join_path(&target, &parts[i + 1..].join("/")), true); + let mut next = if i + 1 < parts.len() { + join_path(&target, &parts[i + 1..].join("/")) + } else { + target + }; + if let Some(suf) = dir_suf { + if !Self::is_dir(&next) { + next.push_str(suf); + } } - return (target, true); + return (next, true); } } } diff --git a/src/ffi.rs b/src/ffi.rs index 10ae29c..5c51a3e 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -40,8 +40,9 @@ pub struct VolatileBuf { /// 外部可改写、必须每读重读后端的控制键（normative：见 spec CABI）。 fn is_volatile(key: &str) -> bool { - (key.starts_with("/vthread/") && key.ends_with("\u{2025}status")) - || (key.starts_with("/lib/") && key.ends_with("/\u{2025}ctime")) + (key.starts_with("/vthread/") && key.contains('\u{2025}')) + || (key.starts_with("/lib/") + && (key.ends_with("/\u{2025}ctime") || key.contains("/vids/"))) } pub struct Handle { @@ -946,8 +947,20 @@ pub extern "C" fn kvspaceDel( .collect(); let code = result_to_code(catch_panic(|| kv.del(&keys)), err, err_cap); if let Some(hd) = unsafe { h.as_mut() } { + let mut is_handoff = false; for k in &keys { hd.cache_invalidate(k); + if (k.starts_with("/lib/") && k.contains("/vids/")) + || k.starts_with("/vthread/\u{2025}vids/") + { + is_handoff = true; + } + } + if is_handoff { + for (_, old) in hd.read_cache.drain() { + hd.retired.push(old); + } + hd.list_cache = None; } } code @@ -1151,9 +1164,10 @@ pub extern "C" fn kvspaceWatch( return 1; } }; - if let Some(old) = hd.read_cache.remove(&key) { - hd.retired.push(old); /* 旧视图保活到 Close（watch 是冷路径） */ + for (_, old) in hd.read_cache.drain() { + hd.retired.push(old); /* Cross-process sync boundary: keep old views alive until Close and force fresh reads. */ } + hd.list_cache = None; hd.read_cache.insert(key.clone(), v.to_wire()); let b = hd.read_cache.get_mut(&key).unwrap(); unsafe { diff --git a/src/fs/kvspace.rs b/src/fs/kvspace.rs index cba4eb2..5d49f2e 100644 --- a/src/fs/kvspace.rs +++ b/src/fs/kvspace.rs @@ -270,13 +270,14 @@ impl FsKVSpace { fn resolve_path(&self, path: &str) -> String { let mut path = path.to_string(); - loop { + for _ in 0..16 { let (resolved, changed) = self.resolve_one(&path); if !changed { return resolved; } path = resolved; } + path } fn resolve_parent(&self, path: &str) -> String { @@ -302,7 +303,19 @@ impl FsKVSpace { if path == PATH_SEP { return (path.to_string(), false); } - let trimmed = path.trim_matches('/'); + let dir_suf = if path.ends_with(OBJ_SEP) { + Some(OBJ_SEP) + } else if path.ends_with(DIR_INDEX_SUF) { + Some(DIR_INDEX_SUF) + } else { + None + }; + let clean = if dir_suf.is_some() { + strip_dir_suf(path) + } else { + path + }; + let trimmed = clean.trim_matches('/'); let parts: Vec<&str> = if trimmed.is_empty() { Vec::new() } else { @@ -315,10 +328,17 @@ impl FsKVSpace { let v = from_wire(&data); if is_ptr(&v) { let target = ptr_target(&v); - if i + 1 < parts.len() { - return (join_path(&target, &parts[i + 1..].join("/")), true); + let mut next = if i + 1 < parts.len() { + join_path(&target, &parts[i + 1..].join("/")) + } else { + target + }; + if let Some(suf) = dir_suf { + if !Self::is_dir_key(&next) { + next.push_str(suf); + } } - return (target, true); + return (next, true); } } } @@ -363,7 +383,12 @@ impl FsKVSpace { if name.ends_with(OBJ_SEP) { children.push(name.trim_end_matches(OBJ_SEP).to_string()); } else if e.path().is_dir() { - children.push(format!("{}/", name)); + let dp = e.path(); + if dp.join(SELF_MARKER).is_file() && !dp.join(DIR_MARKER).is_file() { + children.push(name); + } else { + children.push(format!("{}/", name)); + } } else { children.push(name); } diff --git a/src/kvspace_common.rs b/src/kvspace_common.rs index 998ddd2..3af00e7 100644 --- a/src/kvspace_common.rs +++ b/src/kvspace_common.rs @@ -178,6 +178,7 @@ pub fn watch_value( tick_duration: Duration, ) -> XValue { const SPIN_COUNT: i64 = 100; + let step_cap = tick_duration.min(Duration::from_millis(10)); let mut cur = Duration::ZERO; let mut i: i64 = 0; loop { @@ -191,8 +192,8 @@ pub fn watch_value( } if cur.is_zero() { cur = Duration::from_micros(1); - } else if cur < tick_duration { - cur = (cur * 2).min(tick_duration); + } else if cur < step_cap { + cur = (cur * 2).min(step_cap); } std::thread::sleep(cur); } diff --git a/tests/review_fixes.rs b/tests/review_fixes.rs index 3e75f4c..5de4b8e 100644 --- a/tests/review_fixes.rs +++ b/tests/review_fixes.rs @@ -405,6 +405,58 @@ unsafe fn run_backend_suite(dsn: &str, root: &str) { ); deltree_c(h, &func_dir); + // 10. List(resolve=1) follows Ptr links to bare targets (e.g. /lib//vids -> /vthread/‥vids). + let k_vids_target = format!("{root}/vthread/‥vids"); + let k_vids_entry = format!("{k_vids_target}/89"); + let k_vids_link = format!("{root}/lib/numpy·zeros/vids"); + let w_vids_head = headlenpow::to_wire(ST_SLACK, REF_INLINE, &[], "[]char/utf32·[]char/utf32").unwrap(); + let w_vids_pc = headlenpow::to_wire(ST_SLACK, REF_INLINE, b"[1,0]", "[5]char/utf8").unwrap(); + let w_vids_ptr = new_ptr("[]char/utf32·[]char/utf32", &k_vids_target).to_wire(); + set_raw_checked(h, &k_vids_target, &w_vids_head); + set_raw_checked(h, &k_vids_link, &w_vids_ptr); + set_raw_checked(h, &format!("{k_vids_link}/89"), &w_vids_pc); + + let pfx_vids = CString::new(format!("{k_vids_link}/")).unwrap(); + let mut cnt: i32 = -1; + assert_eq!(kvspaceListLen(h, pfx_vids.as_ptr(), 0, 1, &mut cnt), 0); + assert_eq!(cnt, 1, "ListLen(resolve=1) must list children through a bare-target Ptr"); + let mut name_buf = [0u8; 64]; + let mut name_len: u32 = 0; + assert_eq!( + kvspaceListAt( + h, + pfx_vids.as_ptr(), + 0, + 1, + 0, + name_buf.as_mut_ptr(), + name_buf.len() as u32, + &mut name_len, + ), + 0 + ); + assert_eq!(&name_buf[..name_len as usize], b"89"); + let (rc_ve, p_ve, l_ve) = get_raw_c(h, &k_vids_entry, 0); + assert_eq!(rc_ve, 0); + assert_eq!( + std::slice::from_raw_parts(p_ve, l_ve as usize), + w_vids_pc.as_slice() + ); + + // `·` member enumeration also follows a bare-target Ptr (e.g. /rf/m_link· -> /rf/m_real). + let k_m_real = format!("{root}/m_real"); + let k_m_link = format!("{root}/m_link"); + let w_m_head = headlenpow::to_wire(ST_SHORT, REF_INLINE, &[], "[]char/utf8·int64").unwrap(); + let w_m_val = new_int64(&[42]).to_wire(); + let w_m_ptr = new_ptr("[]char/utf8·int64", &k_m_real).to_wire(); + set_raw_checked(h, &k_m_real, &w_m_head); + set_raw_checked(h, &format!("{k_m_real}·n"), &w_m_val); + set_raw_checked(h, &k_m_link, &w_m_ptr); + let pfx_m = CString::new(format!("{k_m_link}·")).unwrap(); + let mut m_cnt: i32 = -1; + assert_eq!(kvspaceListLen(h, pfx_m.as_ptr(), 0, 1, &mut m_cnt), 0); + assert_eq!(m_cnt, 1, "ListLen(resolve=1) must list `·` members through a bare map Ptr"); + deltree_c(h, root); kvspaceClose(h); } From 89564649ae0aacba58d3b78ba847ed6ad411fd9e Mon Sep 17 00:00:00 2001 From: junyao <1071307515@qq.com> Date: Sat, 31 Oct 2026 21:24:30 +0000 Subject: [PATCH 12/12] ffi: reuse unchanged volatile slot in place and expand ring buffer to 4 slots Avoid advancing or overwriting live VolatileBuf slots when a volatile control key re-reads identical bytes, preventing use-after-free when multiple borrowed Get and GetHead calls interleave within one instruction. --- src/ffi.rs | 21 ++++++++++++--------- tests/read_cache.rs | 14 +++++++++----- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 5c51a3e..dc0cb83 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -32,9 +32,10 @@ use crate::xvalue::{from_wire, from_wire_head, REF_INLINE, REF_PTR, ST_SHORT, ST // op 或 Close」；漏 Close 的兜底由前端 atexit 兑现（见 kvspace.h 写原语注释、spec [[CABI]] // 写侧落盘时机）。调用方从不 free、从不 commit。 -/// 控制键的双缓冲：每次读写入另一块，上一块保活到该 key 下一次读。 +/// Ring buffer for volatile control keys: reuses the current slot in place when unchanged, +/// and advances to the next slot on change so interleaved `Get` / `GetHead` views stay valid. pub struct VolatileBuf { - slots: [Vec; 2], + slots: [Vec; 4], which: usize, } @@ -188,19 +189,21 @@ fn volatile_refresh(hd: &mut Handle, key: &str) -> bool { .volatile_cache .entry(key.to_string()) .or_insert_with(|| VolatileBuf { - slots: [Vec::new(), Vec::new()], + slots: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], which: 0, }); let prev = &vb.slots[vb.which]; - if !prev.is_empty() && prev.as_slice() != raw.as_slice() { - if let Some(func_dir) = key.strip_suffix("/\u{2025}ctime") { - if func_dir.starts_with("/lib/") && func_dir.len() > "/lib/".len() { - ctime_changed_dir = Some(func_dir); + if prev.as_slice() != raw.as_slice() { + if !prev.is_empty() { + if let Some(func_dir) = key.strip_suffix("/\u{2025}ctime") { + if func_dir.starts_with("/lib/") && func_dir.len() > "/lib/".len() { + ctime_changed_dir = Some(func_dir); + } } } + vb.which = (vb.which + 1) & 3; + vb.slots[vb.which] = raw; } - vb.which ^= 1; - vb.slots[vb.which] = raw; } if let Some(func_dir) = ctime_changed_dir { let stale_keys: Vec = hd diff --git a/tests/read_cache.rs b/tests/read_cache.rs index f38b016..c645cdc 100644 --- a/tests/read_cache.rs +++ b/tests/read_cache.rs @@ -353,13 +353,17 @@ fn volatile_status_observed_and_double_buffered() { "previous view must survive one more read" ); - // 再改回 running：A 看到 running；双缓冲只保证最近一次读（p4）的视图有效。 + // Switch back to running: handle A sees running, and repeated unchanged reads keep the same pointer. set_str(b, st, "running"); - let (v3, _p3, _l3) = get_str(a, st).unwrap(); + let (v3, p3, l3) = get_str(a, st).unwrap(); assert_eq!(v3, "running"); - let (v4, p4, l4) = get_str(a, st).unwrap(); - assert_eq!(v4, "running"); - assert_eq!(read_str(p4, l4), "running"); + for _ in 0..6 { + let (vn, pn, ln) = get_str(a, st).unwrap(); + assert_eq!(vn, "running"); + assert_eq!(pn, p3, "unchanged volatile read must keep pointer stable"); + assert_eq!(ln, l3); + } + assert_eq!(read_str(p3, l3), "running"); // 外部删除该 key：volatile 每读重读，必须看到 None。 let del_keys = [ckey(st)];