feat(init): jiebaDictDir and ftsBruteForceByKeysRatio options (#221)

CHANGELOG.md

@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **`ZVec::init()` jieba and FTS tuning options** (#221)
+  - `?string $jiebaDictDir` sets the folder holding `jieba.dict.utf8` and `hmm_model.utf8` for the jieba FTS tokenizer, and `?float $ftsBruteForceByKeysRatio` (0.0–1.0) the point at which an FTS query stops walking posting lists and scores candidates one by one. Upstream default is 0.05. Both take effect only on the first successful `init()` in a process, like every other option.
+  - Read back with `ZVec::getJiebaDictDir()` and `ZVec::getFtsBruteForceByKeysRatio()`. With no option given, the jieba dictionary shipped in `zvec_data/jieba_dict` is still picked up automatically.
+  - `null` rather than `0.0` as the default for the ratio, because 0.0 is a real value upstream accepts.
+  - Both values are validated in PHP **before** any FFI call. Upstream `GlobalConfig::initialize()` sets its initialized flag before validating, so a rejected value would leave the library marked initialized and make every later `init()` a silent no-op; `NAN` also slips through the upstream range check. A `jiebaDictDir` that lacks the dictionary files is rejected for a worse reason: cppjieba calls `abort()`, killing the process with exit code 134 rather than raising a catchable error.
+  - FFI: `zvec_config_data_set_fts_brute_force_by_keys_ratio()`, `zvec_config_data_set_jieba_dict_dir()`, `zvec_global_config_get_fts_brute_force_by_keys_ratio()`, `zvec_global_config_get_jieba_dict_dir()`. The getters go through `global_config_ptr()`, keeping the #215 singleton rule.
+  - Tests: `tests/test_init_fts_options.phpt` (six validation cases that must leave the library uninitialized, plus an end-to-end jieba FTS query using a custom dictionary) and `tests/test_init_fts_defaults.phpt` (upstream defaults).
+
 - **IVF-RaBitQ index type** (#218)
   - `ZVecIndexParams::forIvfRabitq(metricType, nList, totalBits, sampleCount)` builds the new v0.7.0 IVF variant storing RaBitQ-quantized vectors, and `ZVecVectorQuery::setIvfRabitqParams(nprobe)` sends the matching query params. Mirrors the Python `IvfRabitqIndexParam` / `IvfRabitqQueryParam`.
   - New constants `ZVec::INDEX_TYPE_IVF_RABITQ` and `ZVec::QUERY_PARAM_IVF_RABITQ`, both `7`, matching the upstream enum and C ABI.

README.md

@@ -176,9 +176,13 @@ ZVec::init(
     float $bruteForceByKeysRatio = 0.0,
     int $memoryLimitMb = 0,
     ?string $allowedBasePath = null,
-    bool $verboseErrors = false
+    bool $verboseErrors = false,
+    ?string $jiebaDictDir = null,          // folder with jieba.dict.utf8 + hmm_model.utf8
+    ?float $ftsBruteForceByKeysRatio = null  // 0.0-1.0, upstream default 0.05
 ): void
 ZVec::isInitialized(): bool
+ZVec::getFtsBruteForceByKeysRatio(): float
+ZVec::getJiebaDictDir(): string
 ZVec::shutdown(): void
 ZVec::getLastErrorDetails(): array
 ZVec::clearError(): void
@@ -506,6 +510,13 @@ $params = ZVecIndexParams::forFts(
     string[] $filters = ['lowercase'],
     string $extraParams = ''
 ): self
+// Tokenizers: "standard", "ngram", "jieba", "whitespace"
+// Filters:   "lowercase", "ascii_folding", "stemmer"
+// extraParams is a JSON object, e.g. '{"stemmer_lang":"english"}',
+//             '{"ngram_min":2,"ngram_max":3}' or '{"cut_mode":"mix"}'
+// The jieba dictionary ships in zvec_data/jieba_dict next to the library and is
+// used automatically. Lookup order: per-field extraParams jieba_dict_dir, then
+// ZVEC_JIEBA_DICT_DIR, then ZVec::init(jiebaDictDir:), then the bundled copy.
 
 // Invert — keyword-based inverted index
 $params = ZVecIndexParams::forInvert(

ffi/zvec_ffi.cc

@@ -373,6 +373,41 @@ void zvec_config_data_set_brute_force_by_keys_ratio(zvec_config_data_t config, f
     }
 }
 
+void zvec_config_data_set_fts_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio) {
+    if (config) {
+        static_cast<ConfigDataHolder*>(config)->config.fts_brute_force_by_keys_ratio = ratio;
+    }
+}
+
+void zvec_config_data_set_jieba_dict_dir(zvec_config_data_t config, const char* dir) {
+    if (config) {
+        static_cast<ConfigDataHolder*>(config)->config.jieba_dict_dir = dir ? dir : "";
+    }
+}
+
+float zvec_global_config_get_fts_brute_force_by_keys_ratio(void) {
+    auto* gc = global_config_ptr();
+    return gc ? gc->fts_brute_force_by_keys_ratio() : 0.0f;
+}
+
+zvec_status_t zvec_global_config_get_jieba_dict_dir(char* buf, size_t buf_size) {
+    auto* gc = global_config_ptr();
+    if (!gc) {
+        zvec_status_t st = {8, "internal: libzvec GlobalConfig::Instance symbol not found"};
+        SET_FFI_ERROR(st);
+        return st;
+    }
+    if (!buf || buf_size == 0) {
+        zvec_status_t st = {1, "null buffer"};
+        SET_FFI_ERROR(st);
+        return st;
+    }
+    std::string dir = gc->jieba_dict_dir();
+    strncpy(buf, dir.c_str(), buf_size - 1);
+    buf[buf_size - 1] = '\0';
+    return ok_status();
+}
+
 zvec_status_t zvec_ffi_initialize(zvec_config_data_t config) {
     auto* gc = global_config_ptr();
     if (!gc) {

ffi/zvec_ffi.h

@@ -106,6 +106,14 @@ void zvec_config_data_set_query_thread_count(zvec_config_data_t config, uint32_t
 void zvec_config_data_set_optimize_thread_count(zvec_config_data_t config, uint32_t count);
 void zvec_config_data_set_invert_to_forward_scan_ratio(zvec_config_data_t config, float ratio);
 void zvec_config_data_set_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio);
+void zvec_config_data_set_fts_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio);
+// A wrong folder is fatal upstream: cppjieba calls abort() instead of returning
+// a Status, so the path is checked here before it can reach libzvec.
+void zvec_config_data_set_jieba_dict_dir(zvec_config_data_t config, const char* dir);
+
+// Read back the effective process-wide values (valid after init()).
+float zvec_global_config_get_fts_brute_force_by_keys_ratio(void);
+zvec_status_t zvec_global_config_get_jieba_dict_dir(char* buf, size_t buf_size);
 
 zvec_status_t zvec_ffi_initialize(zvec_config_data_t config);
 zvec_status_t zvec_ffi_shutdown(void);

ffi/zvec_ffi_php.h

@@ -77,6 +77,10 @@ void zvec_config_data_set_query_thread_count(zvec_config_data_t config, uint32_t
 void zvec_config_data_set_optimize_thread_count(zvec_config_data_t config, uint32_t count);
 void zvec_config_data_set_invert_to_forward_scan_ratio(zvec_config_data_t config, float ratio);
 void zvec_config_data_set_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio);
+void zvec_config_data_set_fts_brute_force_by_keys_ratio(zvec_config_data_t config, float ratio);
+void zvec_config_data_set_jieba_dict_dir(zvec_config_data_t config, const char* dir);
+float zvec_global_config_get_fts_brute_force_by_keys_ratio(void);
+zvec_status_t zvec_global_config_get_jieba_dict_dir(char* buf, size_t buf_size);
 
 zvec_status_t zvec_ffi_initialize(zvec_config_data_t config);
 zvec_status_t zvec_ffi_shutdown(void);

src/ZVec.php

@@ -1610,12 +1610,41 @@ public static function init(
         int $memoryLimitMb = 0,
         ?string $allowedBasePath = null,
         bool $verboseErrors = false,
+        ?string $jiebaDictDir = null,
+        ?float $ftsBruteForceByKeysRatio = null,
     ): void {
         self::$verboseErrors = $verboseErrors;
 
         if ($allowedBasePath !== null && !is_dir($allowedBasePath)) {
             throw new ZVecException("Allowed base path does not exist: {$allowedBasePath}");
         }
+
+        // Validate before any FFI call. Upstream GlobalConfig::initialize() sets
+        // its "initialized" flag *before* validating, so a value it rejects still
+        // leaves the library marked initialized and every later init() silently
+        // does nothing. NAN also passes the upstream range check.
+        if ($ftsBruteForceByKeysRatio !== null
+            && (is_nan($ftsBruteForceByKeysRatio) || $ftsBruteForceByKeysRatio < 0.0 || $ftsBruteForceByKeysRatio > 1.0)
+        ) {
+            // var_export(), not interpolation: a NAN would emit a PHP warning here.
+            throw new ZVecException(sprintf(
+                'ftsBruteForceByKeysRatio must be between 0 and 1, got: %s',
+                var_export($ftsBruteForceByKeysRatio, true)
+            ));
+        }
+        // A bad jieba folder is fatal: cppjieba calls abort() rather than
+        // returning a Status, killing the process with exit code 134.
+        if ($jiebaDictDir !== null) {
+            if ($jiebaDictDir === '' || !is_dir($jiebaDictDir)) {
+                throw new ZVecException("jiebaDictDir does not exist: {$jiebaDictDir}");
+            }
+            foreach (['jieba.dict.utf8', 'hmm_model.utf8'] as $file) {
+                if (!is_file($jiebaDictDir . '/' . $file)) {
+                    throw new ZVecException("jiebaDictDir is missing {$file}: {$jiebaDictDir}");
+                }
+            }
+        }
+
         self::$allowedBasePath = $allowedBasePath;
 
         $ffi = self::ffi();
@@ -1642,6 +1671,14 @@ public static function init(
         if ($memoryLimitMb > 0) {
             $ffi->zvec_config_data_set_memory_limit($configData, $memoryLimitMb * self::BYTES_PER_MB);
         }
+        // null (not 0.0) means "keep the upstream default", because 0.0 is a real
+        // value here that upstream accepts.
+        if ($ftsBruteForceByKeysRatio !== null) {
+            $ffi->zvec_config_data_set_fts_brute_force_by_keys_ratio($configData, $ftsBruteForceByKeysRatio);
+        }
+        if ($jiebaDictDir !== null) {
+            $ffi->zvec_config_data_set_jieba_dict_dir($configData, $jiebaDictDir);
+        }
 
         try {
             self::checkStatus($ffi->zvec_ffi_initialize($configData));
@@ -1662,6 +1699,48 @@ public static function isInitialized(): bool
         return self::ffi()->zvec_ffi_is_initialized() !== 0;
     }
 
+    /**
+     * Effective process-wide FTS candidate-scanning ratio.
+     *
+     * Point at which an FTS query stops walking the posting lists and starts
+     * checking candidates one by one. Separate from
+     * $bruteForceByKeysRatio because scoring one FTS candidate costs more.
+     * Upstream default is 0.05.
+     *
+     * @throws ZVecException On FFI error
+     */
+    public static function getFtsBruteForceByKeysRatio(): float
+    {
+        return self::ffi()->zvec_global_config_get_fts_brute_force_by_keys_ratio();
+    }
+
+    /**
+     * Effective jieba dictionary directory used by the jieba FTS tokenizer.
+     *
+     * Lookup order is per-field extraParams jieba_dict_dir, then the
+     * ZVEC_JIEBA_DICT_DIR environment variable, then this value, then the
+     * dictionary shipped next to the library.
+     *
+     * @throws ZVecException On FFI error
+     */
+    public static function getJiebaDictDir(): string
+    {
+        $ffi = self::ffi();
+        $bufSize = self::PATH_BUFFER_SIZE;
+        while (true) {
+            $buf = $ffi->new("char[$bufSize]");
+            self::checkStatus($ffi->zvec_global_config_get_jieba_dict_dir($buf, $bufSize));
+            $str = FFI::string($buf);
+            if (strlen($str) < $bufSize - 1) {
+                return $str;
+            }
+            $bufSize *= 2;
+            if ($bufSize > self::MAX_STRING_BUFFER_SIZE) {
+                throw new ZVecException('jiebaDictDir string exceeds maximum buffer size of 1 MB');
+            }
+        }
+    }
+
     /**
      * Shut down the zvec library and release global resources.
      *

tests/test_ffi_load.phpt

@@ -48,6 +48,10 @@ $requiredFunctions = [
     'zvec_get_io_backend_type',
     'zvec_get_io_backend_type_name',
     'zvec_get_io_backend_description',
+    'zvec_config_data_set_fts_brute_force_by_keys_ratio',
+    'zvec_config_data_set_jieba_dict_dir',
+    'zvec_global_config_get_fts_brute_force_by_keys_ratio',
+    'zvec_global_config_get_jieba_dict_dir',
     'zvec_get_last_error_details',
     'zvec_clear_error',
     'zvec_error_code_to_string',
@@ -137,7 +141,7 @@ try {
 echo "DONE\n";
 ?>
 --EXPECT--
-All 65 FFI symbols resolved successfully
+All 69 FFI symbols resolved successfully
 No FFI::cdef() inline string found in src/ZVec.php
 Header file zvec_ffi_php.h is used as source of truth
 Basic create/insert/optimize works

tests/test_init_fts_defaults.phpt

@@ -0,0 +1,24 @@
+--TEST--
+ZVec::init(): upstream defaults for jiebaDictDir and ftsBruteForceByKeysRatio
+--SKIPIF--
+<?php if (!extension_loaded('ffi')) die('skip FFI extension not available'); ?>
+--FILE--
+<?php
+declare(strict_types=1);
+require_once __DIR__ . '/../src/ZVec.php';
+
+// No new options: the getters must report the upstream defaults, and the
+// bundled jieba dictionary must still be picked up automatically.
+ZVec::init(logType: ZVec::LOG_CONSOLE, logLevel: ZVec::LOG_WARN);
+
+echo 'ratio: ' . sprintf('%.2f', ZVec::getFtsBruteForceByKeysRatio()) . "\n";
+
+// The path is derived from the shared library location via dladdr, so it can
+// contain a "src/../ffi/build" segment; match only the tail.
+echo 'jieba dict: ' . (str_ends_with(ZVec::getJiebaDictDir(), '/zvec_data/jieba_dict') ? 'bundled' : 'other') . "\n";
+echo 'is initialized: ' . (ZVec::isInitialized() ? 'yes' : 'no') . "\n";
+?>
+--EXPECT--
+ratio: 0.05
+jieba dict: bundled
+is initialized: yes

tests/test_init_fts_options.phpt

@@ -0,0 +1,122 @@
+--TEST--
+ZVec::init(): jiebaDictDir and ftsBruteForceByKeysRatio options, with getters
+--SKIPIF--
+<?php
+if (!extension_loaded('ffi')) die('skip FFI extension not available');
+
+$found = false;
+foreach (['/../ffi/build/zvec_data/jieba_dict', '/../lib/zvec_data/jieba_dict'] as $d) {
+    if (is_file(__DIR__ . $d . '/jieba.dict.utf8')) { $found = true; break; }
+}
+if (!$found) die('skip bundled jieba dictionary not found');
+if (getenv('ZVEC_JIEBA_DICT_DIR') !== false) die('skip ZVEC_JIEBA_DICT_DIR overrides the bundled dictionary');
+?>
+--FILE--
+<?php
+declare(strict_types=1);
+require_once __DIR__ . '/../src/ZVec.php';
+
+// Validation must reject bad values *before* any FFI call: upstream
+// GlobalConfig::initialize() flips its "initialized" flag before validating, so a
+// value it rejects would still leave the library marked initialized and every
+// later init() would silently do nothing.
+$rejected = [];
+foreach ([
+    'too high' => 1.5,
+    'negative' => -0.1,
+    'NaN' => NAN,
+] as $label => $ratio) {
+    try {
+        ZVec::init(ftsBruteForceByKeysRatio: $ratio);
+        echo "$label: NOT REJECTED\n";
+    } catch (ZVecException $e) {
+        echo "$label: rejected\n";
+    }
+}
+
+foreach ([
+    'empty dir' => '',
+    'missing dir' => '/nonexistent/zvec/jieba',
+] as $label => $dir) {
+    try {
+        ZVec::init(jiebaDictDir: $dir);
+        echo "$label: NOT REJECTED\n";
+    } catch (ZVecException $e) {
+        echo "$label: rejected\n";
+    }
+}
+
+// A folder that exists but lacks the dictionary files would be fatal upstream:
+// cppjieba calls abort() instead of returning a Status, killing the process.
+$emptyDir = __DIR__ . '/../test_dbs/jieba_empty_' . uniqid();
+mkdir($emptyDir, 0777, true);
+try {
+    ZVec::init(jiebaDictDir: $emptyDir);
+    echo "incomplete dir: NOT REJECTED\n";
+} catch (ZVecException $e) {
+    echo "incomplete dir: rejected\n";
+} finally {
+    exec('rm -rf ' . escapeshellarg($emptyDir));
+}
+
+echo 'pre-init: ' . (ZVec::isInitialized() ? '1' : '0') . "\n";
+
+// Now init for real with a *copy* of the bundled dictionary, so the test proves
+// a custom folder is honoured rather than the bundled default.
+$customDir = __DIR__ . '/../test_dbs/jieba_custom_' . uniqid();
+mkdir($customDir, 0777, true);
+$source = is_file(__DIR__ . '/../lib/zvec_data/jieba_dict/jieba.dict.utf8')
+    ? __DIR__ . '/../lib/zvec_data/jieba_dict'
+    : __DIR__ . '/../ffi/build/zvec_data/jieba_dict';
+foreach (['jieba.dict.utf8', 'hmm_model.utf8'] as $file) {
+    copy("{$source}/{$file}", "{$customDir}/{$file}");
+}
+
+$path = __DIR__ . '/../test_dbs/init_fts_' . uniqid();
+try {
+    ZVec::init(
+        logType: ZVec::LOG_CONSOLE,
+        logLevel: ZVec::LOG_WARN,
+        jiebaDictDir: $customDir,
+        ftsBruteForceByKeysRatio: 0.2,
+    );
+
+    // The value is stored as a C float, so compare as a string, not with ===.
+    echo 'ratio: ' . sprintf('%.2f', ZVec::getFtsBruteForceByKeysRatio()) . "\n";
+    echo 'dir matches: ' . (ZVec::getJiebaDictDir() === $customDir ? 'yes' : 'no') . "\n";
+
+    // End to end: an FTS index using the jieba tokenizer without a per-field
+    // jieba_dict_dir, so the global value is what actually resolves it.
+    $schema = new ZVecSchema('jieba_init_test');
+    $schema->addString('body')
+        ->addVectorFp32('vec', dimension: 4, metricType: ZVecSchema::METRIC_IP);
+
+    $collection = ZVec::create($path, $schema);
+    $collection->createIndex('body', ZVecIndexParams::forFts(tokenizer: 'jieba', filters: []));
+    $collection->insert(
+        (new ZVecDoc('j1'))->setString('body', '我来到北京清华大学')->setVectorFp32('vec', [1.0, 0.0, 0.0, 0.0])
+    );
+    $collection->insert(
+        (new ZVecDoc('j2'))->setString('body', '他来到了网易杭研大厦')->setVectorFp32('vec', [0.0, 1.0, 0.0, 0.0])
+    );
+    $collection->flush();
+
+    $query = (new ZVecVectorQuery('body', []))->setTopk(10)->setFts('body', '北京');
+    $hits = array_map(static fn(ZVecDoc $d): string => $d->getPk(), $collection->queryVector($query));
+    echo 'fts hit: ' . implode(',', $hits) . "\n";
+} finally {
+    exec('rm -rf ' . escapeshellarg($path));
+    exec('rm -rf ' . escapeshellarg($customDir));
+}
+?>
+--EXPECT--
+too high: rejected
+negative: rejected
+NaN: rejected
+empty dir: rejected
+missing dir: rejected
+incomplete dir: rejected
+pre-init: 0
+ratio: 0.20
+dir matches: yes
+fts hit: j1