From 4ae380e6a8e77aa10821cdb49985f84485bc7c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=C5=82as=20Piotr?= Date: Mon, 28 Sep 2026 20:29:57 +0200 Subject: [PATCH] feat: expose the DiskANN I/O backend (#224) zvec picks an I/O backend for DiskANN disk reads on first use: on Linux it tries io_uring, then libaio, then falls back to synchronous pread(); macOS arm64 always uses pread(). The choice dominates DiskANN throughput -- pread on Linux usually means libaio simply is not installed -- and until now there was no way to observe it from PHP. Three static getters on ZVec, next to getVersion() and following the same pattern the Python SDK uses for its module-level zvec.io_backend_type(): ZVec::getIoBackendType(): int ZVec::getIoBackendTypeName(int $type): string ZVec::getIoBackendDescription(): string plus IO_BACKEND_PREAD / IO_BACKEND_LIBAIO / IO_BACKEND_IO_URING (0/1/2), the same values as the upstream C ABI. The description is where upstream explains how to install an async backend when pread is in use. These are deliberately static rather than per-collection: the value is process-wide, it is not part of GlobalConfig, and upstream probes it lazily, so the getters work without ZVec::init() at all. The test asserts that. Adhering to the #215 singleton rule: the adapter includes only the public zvec/ailego/io/io_backend.h and calls upstream's exported current_io_backend_type() / current_io_backend_description(), which are compiled inside libzvec, so the IOBackend singleton is created only there. The internal io_backend_def.h is not part of the SDK and IOBackend::Instance() is never referenced from this module. The comment in ffi/zvec_ffi.cc records why, so it does not get "fixed" later. $ nm -C ffi/build/libzvec_ffi.so | grep -c 'IOBackend::Instance' 0 $ nm -D --defined-only ffi/build/libzvec_ffi.so | grep io_backend zvec_get_io_backend_description zvec_get_io_backend_type zvec_get_io_backend_type_name No status to check: these return plain values, matching getVersion(), so self::checkStatus() is not involved and FFI::free() is not needed -- the C side owns the memory (a static literal for the name, a thread-local buffer for the description). Tests: 192/192, 0 skipped, 0 failed, 2 expected fail. The new tests/test_io_backend.phpt covers the constants, the name mapping including the "unknown" fallback, that the value is valid before init(), that the description mentions the reported backend, the macOS pread rule, and that the cached value is stable across repeated calls and across init(). tests/test_ffi_load.phpt gained the three symbols (60 -> 63). --- CHANGELOG.md | 6 ++++ README.md | 7 ++++ ffi/zvec_ffi.cc | 27 ++++++++++++++++ ffi/zvec_ffi.h | 12 +++++++ ffi/zvec_ffi_php.h | 3 ++ src/ZVec.php | 66 ++++++++++++++++++++++++++++++++++++++ tests/test_ffi_load.phpt | 5 ++- tests/test_io_backend.phpt | 50 +++++++++++++++++++++++++++++ 8 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 tests/test_io_backend.phpt diff --git a/CHANGELOG.md b/CHANGELOG.md index dad9fd0..df55800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **DiskANN I/O backend introspection** (#224) + - `ZVec::getIoBackendType()`, `ZVec::getIoBackendTypeName(int $type)` and `ZVec::getIoBackendDescription()` report the I/O backend zvec picked for DiskANN disk reads. Linux tries `io_uring`, then `libaio`, and falls back to synchronous `pread()`; macOS always uses `pread()`. The value matters a lot for DiskANN throughput and previously could not be observed at all. + - The choice is process-wide and resolved lazily, so the getters work without `ZVec::init()`. New constants `ZVec::IO_BACKEND_PREAD`, `IO_BACKEND_LIBAIO` and `IO_BACKEND_IO_URING` (`0`/`1`/`2`), matching the upstream C ABI. + - FFI: `zvec_get_io_backend_type()`, `zvec_get_io_backend_type_name()` and `zvec_get_io_backend_description()`. The adapter calls upstream's exported `current_io_backend_*()` functions and keeps the #215 singleton rule — the internal `io_backend_def.h` is not part of the SDK and `IOBackend::Instance()` is never referenced here. + - Test: `tests/test_io_backend.phpt` (constant values, name mapping including the `unknown` fallback, value valid before `init()`, description matching the reported name, macOS pread rule, stability across repeated calls and across `init()`). + - **Full-Text Search (FTS) support** (#180) - `ZVecIndexParams::forFts(tokenizer, filters, extraParams)` builds a full-text index over a STRING column; mirrors the official Go SDK `NewFTSIndexParams`. Defaults to the `standard` tokenizer with the `lowercase` filter. - `ZVecVectorQuery::setFts(fieldName, queryString, matchString, defaultOperator)` runs an FTS query. `defaultOperator` accepts `ZVec::FTS_OPERATOR_OR` (default) or `FTS_OPERATOR_AND`, case-insensitively. diff --git a/README.md b/README.md index b6fdb0b..6367cd6 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,9 @@ ZVec::checkVersion(int $major, int $minor, int $patch): bool ZVec::getVersionMajor(): int ZVec::getVersionMinor(): int ZVec::getVersionPatch(): int +ZVec::getIoBackendType(): int // ZVec::IO_BACKEND_PREAD|LIBAIO|IO_URING +ZVec::getIoBackendTypeName(int $type): string // "pread" | "libaio" | "io_uring" | "unknown" +ZVec::getIoBackendDescription(): string // DiskANN I/O backend, with install hints on Linux // Collection lifecycle (static factories) $collection = ZVec::create(string $path, ZVecSchema $schema, bool $readOnly = false, bool $enableMmap = true, int $maxBufferSize = 67108864): self @@ -483,6 +486,9 @@ $params = ZVecIndexParams::forDiskAnn( int $pqChunkNum = 0, int $quantizeType = QUANTIZE_UNDEFINED ): self +// On Linux, check ZVec::getIoBackendType(). IO_BACKEND_PREAD means io_uring and +// libaio are both unavailable, so DiskANN reads are synchronous; install libaio +// (libaio1t64 on Ubuntu 24.04+) for async I/O. // Full-Text Search — inverted index over a STRING column $params = ZVecIndexParams::forFts( @@ -875,6 +881,7 @@ See `tasks/done/` for detailed planning documents. - [x] Array field types (STRING, BOOL, INT32, INT64, UINT32, UINT64, FLOAT, DOUBLE) - [x] Group-by vector query builder (`ZVecGroupByVectorQuery`) - [x] Version API (`getVersion()`, `checkVersion()`) +- [x] DiskANN I/O backend introspection (`getIoBackendType()`, `getIoBackendDescription()`) - [x] `allowedBasePath` security restriction in `init()` - [x] Verbose error details with file/line info - [x] Collection lifecycle options via `getOptions()` diff --git a/ffi/zvec_ffi.cc b/ffi/zvec_ffi.cc index dda34d3..f2de818 100644 --- a/ffi/zvec_ffi.cc +++ b/ffi/zvec_ffi.cc @@ -15,6 +15,7 @@ #include #include #include +#include using namespace zvec; @@ -139,6 +140,32 @@ int zvec_get_version_patch(void) { return kVersionPatch; } +// DiskANN I/O backend. current_io_backend_type() and +// current_io_backend_description() are ordinary functions compiled inside +// libzvec; the IOBackend singleton behind them is created only there, so this +// keeps the #215 singleton rule. Do NOT include the internal +// zvec/ailego/io/io_backend_def.h and do not reference IOBackend::Instance() +// from this module: that header is not part of the SDK, and instantiating the +// singleton here would build a second copy destroyed twice at exit. +int zvec_get_io_backend_type(void) { + return static_cast(zvec::ailego::current_io_backend_type()); +} + +const char* zvec_get_io_backend_type_name(int type) { + switch (type) { + case ZVEC_IO_BACKEND_PREAD: return "pread"; + case ZVEC_IO_BACKEND_LIBAIO: return "libaio"; + case ZVEC_IO_BACKEND_IO_URING: return "io_uring"; + default: return "unknown"; + } +} + +const char* zvec_get_io_backend_description(void) { + static thread_local std::string buf; + buf = zvec::ailego::current_io_backend_description(); + return buf.c_str(); +} + static MetricType to_metric_type(uint32_t v) { switch (v) { case 1: return MetricType::L2; diff --git a/ffi/zvec_ffi.h b/ffi/zvec_ffi.h index d847a41..b0afda4 100644 --- a/ffi/zvec_ffi.h +++ b/ffi/zvec_ffi.h @@ -66,6 +66,18 @@ int zvec_get_version_major(void); int zvec_get_version_minor(void); int zvec_get_version_patch(void); +// DiskANN I/O backend (process-wide, probed lazily; no zvec_init() needed). +// Values match zvec::ailego::IOBackendType and are part of the upstream C ABI. +#define ZVEC_IO_BACKEND_PREAD 0 +#define ZVEC_IO_BACKEND_LIBAIO 1 +#define ZVEC_IO_BACKEND_IO_URING 2 +int zvec_get_io_backend_type(void); +// Returns a static string literal owned by the adapter: "pread", "libaio", +// "io_uring", or "unknown" for any other value. +const char* zvec_get_io_backend_type_name(int type); +// Returns a thread_local buffer, overwritten by the next call in this thread. +const char* zvec_get_io_backend_description(void); + // Global init (call once before any other operation) // log_type: 0=console, 1=file // log_level: 0=DEBUG, 1=INFO, 2=WARN, 3=ERROR, 4=FATAL diff --git a/ffi/zvec_ffi_php.h b/ffi/zvec_ffi_php.h index 0fa414f..f63ecdd 100644 --- a/ffi/zvec_ffi_php.h +++ b/ffi/zvec_ffi_php.h @@ -50,6 +50,9 @@ int zvec_check_version(int major, int minor, int patch); int zvec_get_version_major(void); int zvec_get_version_minor(void); int zvec_get_version_patch(void); +int zvec_get_io_backend_type(void); +const char* zvec_get_io_backend_type_name(int type); +const char* zvec_get_io_backend_description(void); zvec_status_t zvec_init(int log_type, int log_level, const char* log_dir, const char* log_basename, diff --git a/src/ZVec.php b/src/ZVec.php index 6277cdd..0ae8832 100644 --- a/src/ZVec.php +++ b/src/ZVec.php @@ -995,6 +995,35 @@ public function fetch(array|string|bool ...$args): array */ public const QUERY_PARAM_DISKANN = 6; + /** + * DiskANN I/O backend: synchronous pread() reads. + * + * Always used on macOS. On Linux it means neither io_uring nor libaio + * could be loaded, so DiskANN disk reads are synchronous. + * + * Value: 0 + */ + public const IO_BACKEND_PREAD = 0; + + /** + * DiskANN I/O backend: libaio. + * + * Loaded at runtime via dlopen(), so it depends on libaio being installed + * on the host. + * + * Value: 1 + */ + public const IO_BACKEND_LIBAIO = 1; + + /** + * DiskANN I/O backend: io_uring via raw kernel syscalls. + * + * Preferred on Linux; requires kernel 5.1+ and no extra dependency. + * + * Value: 2 + */ + public const IO_BACKEND_IO_URING = 2; + /** * Log destination: Console (stderr). * @@ -1696,6 +1725,43 @@ public static function getVersionPatch(): int return self::ffi()->zvec_get_version_patch(); } + /** + * DiskANN I/O backend in use, one of the ZVec::IO_BACKEND_* constants. + * + * The choice is process-wide and resolved lazily on the first call, so + * this works without ZVec::init(). Linux prefers io_uring, then libaio, + * then falls back to synchronous pread(); macOS always uses pread. + * + * @throws ZVecException On FFI error + */ + public static function getIoBackendType(): int + { + return self::ffi()->zvec_get_io_backend_type(); + } + + /** + * Name of a ZVec::IO_BACKEND_* value, or "unknown" for anything else. + * + * @throws ZVecException On FFI error + */ + public static function getIoBackendTypeName(int $type): string + { + return self::ffi()->zvec_get_io_backend_type_name($type); + } + + /** + * Human-readable description of the active I/O backend. + * + * On Linux the pread variant also explains how to install an async + * backend, so this is the place to look when DiskANN reads seem slow. + * + * @throws ZVecException On FFI error + */ + public static function getIoBackendDescription(): string + { + return self::ffi()->zvec_get_io_backend_description(); + } + /** * Query using a native ZVecVectorQuery object. * diff --git a/tests/test_ffi_load.phpt b/tests/test_ffi_load.phpt index 5873ddd..1042cfa 100644 --- a/tests/test_ffi_load.phpt +++ b/tests/test_ffi_load.phpt @@ -45,6 +45,9 @@ $requiredFunctions = [ 'zvec_get_version_major', 'zvec_get_version_minor', 'zvec_get_version_patch', + 'zvec_get_io_backend_type', + 'zvec_get_io_backend_type_name', + 'zvec_get_io_backend_description', 'zvec_get_last_error_details', 'zvec_clear_error', 'zvec_error_code_to_string', @@ -132,7 +135,7 @@ try { echo "DONE\n"; ?> --EXPECT-- -All 60 FFI symbols resolved successfully +All 63 FFI symbols resolved successfully No FFI::cdef() inline string found in src/ZVec.php Header file zvec_ffi_php.h is used as source of truth Basic create/insert/optimize works diff --git a/tests/test_io_backend.phpt b/tests/test_io_backend.phpt new file mode 100644 index 0000000..0821263 --- /dev/null +++ b/tests/test_io_backend.phpt @@ -0,0 +1,50 @@ +--TEST-- +DiskANN I/O backend: getIoBackendType, getIoBackendTypeName, getIoBackendDescription +--SKIPIF-- + +--FILE-- + +--EXPECT-- +constants: 0,1,2 +names: pread,libaio,io_uring,unknown +type valid: yes +description non-empty: yes +description matches name: yes +platform check: ok +stable: yes +after init: yes +PASS