diff --git a/CHANGELOG.md b/CHANGELOG.md index 523bf5d..aacff9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`ZVec::init()` jieba and FTS tuning options** (#221) + - `?string $jiebaDictDir` sets the folder holding `jieba.dict.utf8` and `hmm_model.utf8` for the jieba FTS tokenizer, and `?float $ftsBruteForceByKeysRatio` (0.0–1.0) the point at which an FTS query stops walking posting lists and scores candidates one by one. Upstream default is 0.05. Both take effect only on the first successful `init()` in a process, like every other option. + - Read back with `ZVec::getJiebaDictDir()` and `ZVec::getFtsBruteForceByKeysRatio()`. With no option given, the jieba dictionary shipped in `zvec_data/jieba_dict` is still picked up automatically. + - `null` rather than `0.0` as the default for the ratio, because 0.0 is a real value upstream accepts. + - Both values are validated in PHP **before** any FFI call. Upstream `GlobalConfig::initialize()` sets its initialized flag before validating, so a rejected value would leave the library marked initialized and make every later `init()` a silent no-op; `NAN` also slips through the upstream range check. A `jiebaDictDir` that lacks the dictionary files is rejected for a worse reason: cppjieba calls `abort()`, killing the process with exit code 134 rather than raising a catchable error. + - FFI: `zvec_config_data_set_fts_brute_force_by_keys_ratio()`, `zvec_config_data_set_jieba_dict_dir()`, `zvec_global_config_get_fts_brute_force_by_keys_ratio()`, `zvec_global_config_get_jieba_dict_dir()`. The getters go through `global_config_ptr()`, keeping the #215 singleton rule. + - Tests: `tests/test_init_fts_options.phpt` (six validation cases that must leave the library uninitialized, plus an end-to-end jieba FTS query using a custom dictionary) and `tests/test_init_fts_defaults.phpt` (upstream defaults). + - **IVF-RaBitQ index type** (#218) - `ZVecIndexParams::forIvfRabitq(metricType, nList, totalBits, sampleCount)` builds the new v0.7.0 IVF variant storing RaBitQ-quantized vectors, and `ZVecVectorQuery::setIvfRabitqParams(nprobe)` sends the matching query params. Mirrors the Python `IvfRabitqIndexParam` / `IvfRabitqQueryParam`. - New constants `ZVec::INDEX_TYPE_IVF_RABITQ` and `ZVec::QUERY_PARAM_IVF_RABITQ`, both `7`, matching the upstream enum and C ABI. diff --git a/README.md b/README.md index 7766027..a0b49e0 100644 --- a/README.md +++ b/README.md @@ -176,9 +176,13 @@ ZVec::init( float $bruteForceByKeysRatio = 0.0, int $memoryLimitMb = 0, ?string $allowedBasePath = null, - bool $verboseErrors = false + bool $verboseErrors = false, + ?string $jiebaDictDir = null, // folder with jieba.dict.utf8 + hmm_model.utf8 + ?float $ftsBruteForceByKeysRatio = null // 0.0-1.0, upstream default 0.05 ): void ZVec::isInitialized(): bool +ZVec::getFtsBruteForceByKeysRatio(): float +ZVec::getJiebaDictDir(): string ZVec::shutdown(): void ZVec::getLastErrorDetails(): array ZVec::clearError(): void @@ -506,6 +510,13 @@ $params = ZVecIndexParams::forFts( string[] $filters = ['lowercase'], string $extraParams = '' ): self +// Tokenizers: "standard", "ngram", "jieba", "whitespace" +// Filters: "lowercase", "ascii_folding", "stemmer" +// extraParams is a JSON object, e.g. '{"stemmer_lang":"english"}', +// '{"ngram_min":2,"ngram_max":3}' or '{"cut_mode":"mix"}' +// The jieba dictionary ships in zvec_data/jieba_dict next to the library and is +// used automatically. Lookup order: per-field extraParams jieba_dict_dir, then +// ZVEC_JIEBA_DICT_DIR, then ZVec::init(jiebaDictDir:), then the bundled copy. // Invert — keyword-based inverted index $params = ZVecIndexParams::forInvert( diff --git a/ffi/zvec_ffi.cc b/ffi/zvec_ffi.cc index bdfd380..c516903 100644 --- a/ffi/zvec_ffi.cc +++ b/ffi/zvec_ffi.cc @@ -373,6 +373,41 @@ void zvec_config_data_set_brute_force_by_keys_ratio(zvec_config_data_t config, f } } +void zvec_config_data_set_fts_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio) { + if (config) { + static_cast(config)->config.fts_brute_force_by_keys_ratio = ratio; + } +} + +void zvec_config_data_set_jieba_dict_dir(zvec_config_data_t config, const char* dir) { + if (config) { + static_cast(config)->config.jieba_dict_dir = dir ? dir : ""; + } +} + +float zvec_global_config_get_fts_brute_force_by_keys_ratio(void) { + auto* gc = global_config_ptr(); + return gc ? gc->fts_brute_force_by_keys_ratio() : 0.0f; +} + +zvec_status_t zvec_global_config_get_jieba_dict_dir(char* buf, size_t buf_size) { + auto* gc = global_config_ptr(); + if (!gc) { + zvec_status_t st = {8, "internal: libzvec GlobalConfig::Instance symbol not found"}; + SET_FFI_ERROR(st); + return st; + } + if (!buf || buf_size == 0) { + zvec_status_t st = {1, "null buffer"}; + SET_FFI_ERROR(st); + return st; + } + std::string dir = gc->jieba_dict_dir(); + strncpy(buf, dir.c_str(), buf_size - 1); + buf[buf_size - 1] = '\0'; + return ok_status(); +} + zvec_status_t zvec_ffi_initialize(zvec_config_data_t config) { auto* gc = global_config_ptr(); if (!gc) { diff --git a/ffi/zvec_ffi.h b/ffi/zvec_ffi.h index d2702f1..2116dc3 100644 --- a/ffi/zvec_ffi.h +++ b/ffi/zvec_ffi.h @@ -106,6 +106,14 @@ void zvec_config_data_set_query_thread_count(zvec_config_data_t config, uint32_t void zvec_config_data_set_optimize_thread_count(zvec_config_data_t config, uint32_t count); void zvec_config_data_set_invert_to_forward_scan_ratio(zvec_config_data_t config, float ratio); void zvec_config_data_set_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio); +void zvec_config_data_set_fts_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio); +// A wrong folder is fatal upstream: cppjieba calls abort() instead of returning +// a Status, so the path is checked here before it can reach libzvec. +void zvec_config_data_set_jieba_dict_dir(zvec_config_data_t config, const char* dir); + +// Read back the effective process-wide values (valid after init()). +float zvec_global_config_get_fts_brute_force_by_keys_ratio(void); +zvec_status_t zvec_global_config_get_jieba_dict_dir(char* buf, size_t buf_size); zvec_status_t zvec_ffi_initialize(zvec_config_data_t config); zvec_status_t zvec_ffi_shutdown(void); diff --git a/ffi/zvec_ffi_php.h b/ffi/zvec_ffi_php.h index 1ffbf25..7a1d6e4 100644 --- a/ffi/zvec_ffi_php.h +++ b/ffi/zvec_ffi_php.h @@ -77,6 +77,10 @@ void zvec_config_data_set_query_thread_count(zvec_config_data_t config, uint32_t void zvec_config_data_set_optimize_thread_count(zvec_config_data_t config, uint32_t count); void zvec_config_data_set_invert_to_forward_scan_ratio(zvec_config_data_t config, float ratio); void zvec_config_data_set_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio); +void zvec_config_data_set_fts_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio); +void zvec_config_data_set_jieba_dict_dir(zvec_config_data_t config, const char* dir); +float zvec_global_config_get_fts_brute_force_by_keys_ratio(void); +zvec_status_t zvec_global_config_get_jieba_dict_dir(char* buf, size_t buf_size); zvec_status_t zvec_ffi_initialize(zvec_config_data_t config); zvec_status_t zvec_ffi_shutdown(void); diff --git a/src/ZVec.php b/src/ZVec.php index afe1c5e..1ca1daf 100644 --- a/src/ZVec.php +++ b/src/ZVec.php @@ -1610,12 +1610,41 @@ public static function init( int $memoryLimitMb = 0, ?string $allowedBasePath = null, bool $verboseErrors = false, + ?string $jiebaDictDir = null, + ?float $ftsBruteForceByKeysRatio = null, ): void { self::$verboseErrors = $verboseErrors; if ($allowedBasePath !== null && !is_dir($allowedBasePath)) { throw new ZVecException("Allowed base path does not exist: {$allowedBasePath}"); } + + // Validate before any FFI call. Upstream GlobalConfig::initialize() sets + // its "initialized" flag *before* validating, so a value it rejects still + // leaves the library marked initialized and every later init() silently + // does nothing. NAN also passes the upstream range check. + if ($ftsBruteForceByKeysRatio !== null + && (is_nan($ftsBruteForceByKeysRatio) || $ftsBruteForceByKeysRatio < 0.0 || $ftsBruteForceByKeysRatio > 1.0) + ) { + // var_export(), not interpolation: a NAN would emit a PHP warning here. + throw new ZVecException(sprintf( + 'ftsBruteForceByKeysRatio must be between 0 and 1, got: %s', + var_export($ftsBruteForceByKeysRatio, true) + )); + } + // A bad jieba folder is fatal: cppjieba calls abort() rather than + // returning a Status, killing the process with exit code 134. + if ($jiebaDictDir !== null) { + if ($jiebaDictDir === '' || !is_dir($jiebaDictDir)) { + throw new ZVecException("jiebaDictDir does not exist: {$jiebaDictDir}"); + } + foreach (['jieba.dict.utf8', 'hmm_model.utf8'] as $file) { + if (!is_file($jiebaDictDir . '/' . $file)) { + throw new ZVecException("jiebaDictDir is missing {$file}: {$jiebaDictDir}"); + } + } + } + self::$allowedBasePath = $allowedBasePath; $ffi = self::ffi(); @@ -1642,6 +1671,14 @@ public static function init( if ($memoryLimitMb > 0) { $ffi->zvec_config_data_set_memory_limit($configData, $memoryLimitMb * self::BYTES_PER_MB); } + // null (not 0.0) means "keep the upstream default", because 0.0 is a real + // value here that upstream accepts. + if ($ftsBruteForceByKeysRatio !== null) { + $ffi->zvec_config_data_set_fts_brute_force_by_keys_ratio($configData, $ftsBruteForceByKeysRatio); + } + if ($jiebaDictDir !== null) { + $ffi->zvec_config_data_set_jieba_dict_dir($configData, $jiebaDictDir); + } try { self::checkStatus($ffi->zvec_ffi_initialize($configData)); @@ -1662,6 +1699,48 @@ public static function isInitialized(): bool return self::ffi()->zvec_ffi_is_initialized() !== 0; } + /** + * Effective process-wide FTS candidate-scanning ratio. + * + * Point at which an FTS query stops walking the posting lists and starts + * checking candidates one by one. Separate from + * $bruteForceByKeysRatio because scoring one FTS candidate costs more. + * Upstream default is 0.05. + * + * @throws ZVecException On FFI error + */ + public static function getFtsBruteForceByKeysRatio(): float + { + return self::ffi()->zvec_global_config_get_fts_brute_force_by_keys_ratio(); + } + + /** + * Effective jieba dictionary directory used by the jieba FTS tokenizer. + * + * Lookup order is per-field extraParams jieba_dict_dir, then the + * ZVEC_JIEBA_DICT_DIR environment variable, then this value, then the + * dictionary shipped next to the library. + * + * @throws ZVecException On FFI error + */ + public static function getJiebaDictDir(): string + { + $ffi = self::ffi(); + $bufSize = self::PATH_BUFFER_SIZE; + while (true) { + $buf = $ffi->new("char[$bufSize]"); + self::checkStatus($ffi->zvec_global_config_get_jieba_dict_dir($buf, $bufSize)); + $str = FFI::string($buf); + if (strlen($str) < $bufSize - 1) { + return $str; + } + $bufSize *= 2; + if ($bufSize > self::MAX_STRING_BUFFER_SIZE) { + throw new ZVecException('jiebaDictDir string exceeds maximum buffer size of 1 MB'); + } + } + } + /** * Shut down the zvec library and release global resources. * diff --git a/tests/test_ffi_load.phpt b/tests/test_ffi_load.phpt index 23b98a9..ff62c65 100644 --- a/tests/test_ffi_load.phpt +++ b/tests/test_ffi_load.phpt @@ -48,6 +48,10 @@ $requiredFunctions = [ 'zvec_get_io_backend_type', 'zvec_get_io_backend_type_name', 'zvec_get_io_backend_description', + 'zvec_config_data_set_fts_brute_force_by_keys_ratio', + 'zvec_config_data_set_jieba_dict_dir', + 'zvec_global_config_get_fts_brute_force_by_keys_ratio', + 'zvec_global_config_get_jieba_dict_dir', 'zvec_get_last_error_details', 'zvec_clear_error', 'zvec_error_code_to_string', @@ -137,7 +141,7 @@ try { echo "DONE\n"; ?> --EXPECT-- -All 65 FFI symbols resolved successfully +All 69 FFI symbols resolved successfully No FFI::cdef() inline string found in src/ZVec.php Header file zvec_ffi_php.h is used as source of truth Basic create/insert/optimize works diff --git a/tests/test_init_fts_defaults.phpt b/tests/test_init_fts_defaults.phpt new file mode 100644 index 0000000..ad8a736 --- /dev/null +++ b/tests/test_init_fts_defaults.phpt @@ -0,0 +1,24 @@ +--TEST-- +ZVec::init(): upstream defaults for jiebaDictDir and ftsBruteForceByKeysRatio +--SKIPIF-- + +--FILE-- + +--EXPECT-- +ratio: 0.05 +jieba dict: bundled +is initialized: yes diff --git a/tests/test_init_fts_options.phpt b/tests/test_init_fts_options.phpt new file mode 100644 index 0000000..ec58aa0 --- /dev/null +++ b/tests/test_init_fts_options.phpt @@ -0,0 +1,122 @@ +--TEST-- +ZVec::init(): jiebaDictDir and ftsBruteForceByKeysRatio options, with getters +--SKIPIF-- + +--FILE-- + 1.5, + 'negative' => -0.1, + 'NaN' => NAN, +] as $label => $ratio) { + try { + ZVec::init(ftsBruteForceByKeysRatio: $ratio); + echo "$label: NOT REJECTED\n"; + } catch (ZVecException $e) { + echo "$label: rejected\n"; + } +} + +foreach ([ + 'empty dir' => '', + 'missing dir' => '/nonexistent/zvec/jieba', +] as $label => $dir) { + try { + ZVec::init(jiebaDictDir: $dir); + echo "$label: NOT REJECTED\n"; + } catch (ZVecException $e) { + echo "$label: rejected\n"; + } +} + +// A folder that exists but lacks the dictionary files would be fatal upstream: +// cppjieba calls abort() instead of returning a Status, killing the process. +$emptyDir = __DIR__ . '/../test_dbs/jieba_empty_' . uniqid(); +mkdir($emptyDir, 0777, true); +try { + ZVec::init(jiebaDictDir: $emptyDir); + echo "incomplete dir: NOT REJECTED\n"; +} catch (ZVecException $e) { + echo "incomplete dir: rejected\n"; +} finally { + exec('rm -rf ' . escapeshellarg($emptyDir)); +} + +echo 'pre-init: ' . (ZVec::isInitialized() ? '1' : '0') . "\n"; + +// Now init for real with a *copy* of the bundled dictionary, so the test proves +// a custom folder is honoured rather than the bundled default. +$customDir = __DIR__ . '/../test_dbs/jieba_custom_' . uniqid(); +mkdir($customDir, 0777, true); +$source = is_file(__DIR__ . '/../lib/zvec_data/jieba_dict/jieba.dict.utf8') + ? __DIR__ . '/../lib/zvec_data/jieba_dict' + : __DIR__ . '/../ffi/build/zvec_data/jieba_dict'; +foreach (['jieba.dict.utf8', 'hmm_model.utf8'] as $file) { + copy("{$source}/{$file}", "{$customDir}/{$file}"); +} + +$path = __DIR__ . '/../test_dbs/init_fts_' . uniqid(); +try { + ZVec::init( + logType: ZVec::LOG_CONSOLE, + logLevel: ZVec::LOG_WARN, + jiebaDictDir: $customDir, + ftsBruteForceByKeysRatio: 0.2, + ); + + // The value is stored as a C float, so compare as a string, not with ===. + echo 'ratio: ' . sprintf('%.2f', ZVec::getFtsBruteForceByKeysRatio()) . "\n"; + echo 'dir matches: ' . (ZVec::getJiebaDictDir() === $customDir ? 'yes' : 'no') . "\n"; + + // End to end: an FTS index using the jieba tokenizer without a per-field + // jieba_dict_dir, so the global value is what actually resolves it. + $schema = new ZVecSchema('jieba_init_test'); + $schema->addString('body') + ->addVectorFp32('vec', dimension: 4, metricType: ZVecSchema::METRIC_IP); + + $collection = ZVec::create($path, $schema); + $collection->createIndex('body', ZVecIndexParams::forFts(tokenizer: 'jieba', filters: [])); + $collection->insert( + (new ZVecDoc('j1'))->setString('body', '我来到北京清华大学')->setVectorFp32('vec', [1.0, 0.0, 0.0, 0.0]) + ); + $collection->insert( + (new ZVecDoc('j2'))->setString('body', '他来到了网易杭研大厦')->setVectorFp32('vec', [0.0, 1.0, 0.0, 0.0]) + ); + $collection->flush(); + + $query = (new ZVecVectorQuery('body', []))->setTopk(10)->setFts('body', '北京'); + $hits = array_map(static fn(ZVecDoc $d): string => $d->getPk(), $collection->queryVector($query)); + echo 'fts hit: ' . implode(',', $hits) . "\n"; +} finally { + exec('rm -rf ' . escapeshellarg($path)); + exec('rm -rf ' . escapeshellarg($customDir)); +} +?> +--EXPECT-- +too high: rejected +negative: rejected +NaN: rejected +empty dir: rejected +missing dir: rejected +incomplete dir: rejected +pre-init: 0 +ratio: 0.20 +dir matches: yes +fts hit: j1