From 4fc740c69f4ec3df74478785126ba291cd6328e1 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 21 Oct 2026 12:38:46 -0700 Subject: [PATCH 01/47] Add filtered search on the dynamic Vamana search buffer Reuse the MutableBuffer valid/invalid mechanism: a FilteredValidBuilder marks an entry invalid if it is deleted or the predicate rejects its external id. New MutableVamanaIndex::search overload taking a predicate, padding rows with id -1 / +inf and returning per-query found counts. Supplement path checks the predicate before computing the distance. Orchestrator virtual appended last. --- include/svs/index/vamana/dynamic_index.h | 122 +++++++++++++++- include/svs/index/vamana/extensions.h | 26 ++++ include/svs/orchestrators/dynamic_vamana.h | 58 ++++++++ tests/svs/index/vamana/dynamic_index.cpp | 157 +++++++++++++++++++++ 4 files changed, 361 insertions(+), 2 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 9336f82e1..5c75d4445 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -18,6 +18,7 @@ // stdlib #include +#include #include #include @@ -109,11 +110,49 @@ class ValidBuilder { const std::vector& status_; }; +// Like `ValidBuilder`, but an entry is also invalid if `pred(external_id)` is false. +template class FilteredValidBuilder { + public: + FilteredValidBuilder( + const std::vector& status, + const IDTranslator& translator, + const Pred& pred + ) + : status_{status} + , translator_{translator} + , pred_{pred} {} + + template bool is_valid(I internal_id, size_t external_id) const { + return getindex(status_, internal_id) != SlotMetadata::Deleted && + pred_(external_id); + } + + template + PredicatedSearchNeighbor operator()(I i, float distance) const { + bool valid = getindex(status_, i) != SlotMetadata::Deleted && + pred_(translator_.get_external(i)); + return PredicatedSearchNeighbor(i, distance, valid); + } + + private: + const std::vector& status_; + const IDTranslator& translator_; + const Pred& pred_; +}; + +// Data types for which the filtered `search` overload is valid (identity accessor). +template inline constexpr bool filtered_search_supported = false; +template +inline constexpr bool filtered_search_supported> = true; + template class MutableVamanaIndex { friend class MultiMutableVamanaIndex; public: + // Present only on indexes that support the filtered `search` overload. + static constexpr bool has_filtered_search = filtered_search_supported; + // Traits static constexpr bool supports_insertions = true; static constexpr bool supports_deletions = true; @@ -575,11 +614,13 @@ class MutableVamanaIndex { // This is an internal method, mostly used to help implement the batch iterator. ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + template auto greedy_search_closure( + const Builder& builder, GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) const { - return [&, prefetch_parameters]( + return [&, builder, prefetch_parameters]( const auto& query, auto& accessor, auto& distance, auto& buffer ) { // Perform the greedy search using the provided resources. @@ -591,7 +632,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(), + builder, prefetch_parameters, cancel ); @@ -601,6 +642,15 @@ class MutableVamanaIndex { }; } + auto greedy_search_closure( + GreedySearchPrefetchParameters prefetch_parameters, + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + ) const { + return greedy_search_closure( + internal_search_builder(), prefetch_parameters, cancel + ); + } + // Single Search template void search( @@ -667,6 +717,74 @@ class MutableVamanaIndex { translate_to_external(results.indices()); } + /// + /// @brief Filtered top-k search: only live entries whose external id satisfies `pred` + /// are returned. Invalid entries are navigated through but do not count toward the + /// search window. + /// + /// @param pred Callable `bool(size_t external_id)`. Must be thread safe. + /// @param found Resized to `queries.size()`; receives the number of results per query. + /// + /// Rows with fewer than `n_neighbors` results are padded with id `size_t(-1)` and + /// distance `+inf`. Requires `filtered_search_supported`. + /// + template + requires(std::is_invocable_r_v) + void search( + QueryResultView results, + const Queries& queries, + const search_parameters_type& sp, + const Pred& pred, + std::vector& found + ) { + static_assert( + filtered_search_supported, + "Data type needs a custom search path (e.g. reranking)." + ); + found.assign(queries.size(), 0); + threads::parallel_for( + threadpool_, + threads::StaticPartition{queries.size()}, + [&](const auto is, uint64_t SVS_UNUSED(tid)) { + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + auto distance = extensions::per_thread_batch_search_setup(data_, distance_); + auto accessor = data::GetDatumAccessor(); + auto builder = FilteredValidBuilder{status_, translator_, pred}; + // The closure holds `cancel` by reference, so it must outlive the closure. + const lib::DefaultPredicate cancel = lib::Returns(lib::Const()); + auto search = greedy_search_closure(builder, prefetch_parameters, cancel); + + for (auto i : is) { + auto query = queries.get_datum(i); + search(query, accessor, distance, buffer); + extensions::check_and_supplement_search_buffer( + *this, buffer, query, builder + ); + // Everything in the buffer is valid here (cleanup() ran, and + // supplemented entries are checked before insertion). + size_t n = std::min(num_neighbors, buffer.size()); + for (size_t j = 0; j < n; ++j) { + results.index(i, j) = translate_internal_id(buffer[j].id()); + results.distance(i, j) = buffer[j].distance(); + } + for (size_t j = n; j < num_neighbors; ++j) { + results.index(i, j) = static_cast(-1); + results.distance(i, j) = std::numeric_limits::infinity(); + } + found[i] = n; + } + } + ); + } + /// /// @brief Return a unique instance of the distance function. /// diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 96ffeaa06..5b73cf7e9 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -449,6 +449,32 @@ void check_and_supplement_search_buffer( } } +/// Filtered variant: only ids accepted by `builder` are inserted, and the predicate is +/// evaluated before the distance is computed. Does not change the unfiltered path above. +template +void check_and_supplement_search_buffer( + const Index& index, + SearchBuffer& search_buffer, + const Query& query, + const Builder& builder +) { + if (search_buffer.valid() < search_buffer.target_window() && + search_buffer.valid() < index.size()) { + search_buffer.sort(); + for (auto external_id : index.external_ids()) { + auto internal_id = index.translate_external_id(external_id); + if (!builder.is_valid(internal_id, external_id)) { + continue; + } + auto dist = index.get_distance(external_id, query); + search_buffer.insert(builder(internal_id, dist)); + if (search_buffer.valid() >= search_buffer.target_window()) { + break; + } + } + } +} + /// Customization point object for processing single queries. inline constexpr VamanaSingleSearchType single_search{}; diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 067c7c486..e82c5c0f9 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -22,6 +22,7 @@ #include "svs/orchestrators/vamana.h" // stdlib +#include #include namespace svs { @@ -56,6 +57,17 @@ class DynamicVamanaInterface : public VamanaInterface { // Check if the index has been consolidated - no `Deleted` entries remain. virtual bool is_consolidated() const = 0; + + // Filtered top-k on the search buffer; `filter` takes an external id. Returns false, + // leaving `results` untouched, if the data type does not support it. + // Keep last: new virtuals must be appended. + virtual bool filtered_search( + svs::QueryResultView results, + AnonymousArray<2> queries, + const index::vamana::VamanaSearchParameters& search_parameters, + const std::function& filter, + std::vector& found + ) = 0; }; template @@ -109,6 +121,31 @@ class DynamicVamanaImpl : public VamanaImpl results, + AnonymousArray<2> queries, + const index::vamana::VamanaSearchParameters& search_parameters, + const std::function& filter, + std::vector& found + ) override { + if constexpr (requires { requires Impl::has_filtered_search; }) { + lib::match( + QueryTypes{}, + queries.type(), + [&](lib::Type SVS_UNUSED(type)) { + const auto view = data::ConstSimpleDataView(queries); + impl().search(results, view, search_parameters, filter, found); + }, + [&](svs::DataType data_type) { + throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); + } + ); + return true; + } else { + return false; + } + } }; // Forward Declaractions. @@ -609,6 +646,27 @@ class DynamicVamana : public manager::IndexManager { /// @brief Check if the index has been consolidated - no `Deleted` entries remain. bool is_consolidated() const { return impl_->is_consolidated(); } + + /// + /// @brief Filtered top-k search on the search buffer, see + /// ``MutableVamanaIndex::search`` (filtered overload). + /// + /// @param filter Takes an external id. + /// @param found Receives the number of results per query. Rows are padded with id + /// ``size_t(-1)`` and distance ``+inf``. + /// @return False if the data type is unsupported, in which case nothing is written. + template + bool filtered_search( + QueryResultView results, + data::ConstSimpleDataView queries, + const VamanaSearchParameters& search_parameters, + const std::function& filter, + std::vector& found + ) { + return impl_->filtered_search( + results, AnonymousArray<2>(queries), search_parameters, filter, found + ); + } }; /// diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 94daddcf1..7a7fadabe 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -23,11 +23,14 @@ #include #include #include +#include #include #include +#include #include #include #include +#include // svs #include "svs/core/recall.h" @@ -464,3 +467,157 @@ CATCH_TEST_CASE("MutableVamana Index Memory Usage", "[graph_index][dynamic_index const size_t usage = index.get_memory_breakdown().total(); CATCH_REQUIRE(usage == expected_total_bytes); } + +CATCH_TEST_CASE( + "MutableVamana Index Filtered Search", "[graph_index][dynamic_index][filtered]" +) { + const size_t num_threads = 2; + const size_t n = 3000; + const size_t dim = 32; + const size_t num_queries = 50; + const size_t k = 10; + using Distance = svs::distance::DistanceL2; + + std::mt19937 rng(42); + std::normal_distribution normal; + auto data = svs::data::BlockedData(n, dim); + std::vector> raw(n, std::vector(dim)); + for (size_t i = 0; i < n; ++i) { + for (auto& x : raw[i]) { + x = normal(rng); + } + data.set_datum(i, raw[i]); + } + auto queries = svs::data::SimpleData(num_queries, dim); + for (size_t i = 0; i < num_queries; ++i) { + std::vector q(dim); + for (auto& x : q) { + x = normal(rng); + } + queries.set_datum(i, q); + } + + // External ids differ from internal ids. + std::vector ids(n); + std::iota(ids.begin(), ids.end(), 7); + svs::index::vamana::VamanaBuildParameters parameters{1.2, 32, 64, 64, 32, true}; + auto index = svs::index::vamana::MutableVamanaIndex( + parameters, std::move(data), ids, Distance(), num_threads + ); + + auto sp = svs::index::vamana::VamanaSearchParameters{}; + sp.buffer_config_ = svs::index::vamana::SearchBufferConfig{50}; + + // Brute force over live ids passing `pred`. + std::unordered_set deleted; + auto brute_force = [&](size_t q, auto&& pred) { + std::vector> all; + for (size_t i = 0; i < n; ++i) { + size_t e = ids[i]; + if (deleted.contains(e) || !pred(e)) { + continue; + } + float d = 0; + for (size_t j = 0; j < dim; ++j) { + float diff = queries.get_datum(q)[j] - raw[i][j]; + d += diff * diff; + } + all.emplace_back(d, e); + } + std::sort(all.begin(), all.end()); + std::vector out; + for (size_t i = 0; i < std::min(k, all.size()); ++i) { + out.push_back(all[i].second); + } + return out; + }; + + auto check = [&](auto&& pred, bool expect_full) { + auto results = svs::QueryResult(num_queries, k); + std::vector found; + index.search(results.view(), queries.cview(), sp, pred, found); + CATCH_REQUIRE(found.size() == num_queries); + size_t hits = 0, total = 0; + for (size_t q = 0; q < num_queries; ++q) { + auto gt = brute_force(q, pred); + CATCH_REQUIRE(found[q] <= k); + if (expect_full) { + CATCH_REQUIRE(found[q] == k); + } + for (size_t j = 0; j < found[q]; ++j) { + size_t id = results.index(q, j); + CATCH_REQUIRE(pred(id)); + CATCH_REQUIRE(!deleted.contains(id)); + hits += std::count(gt.begin(), gt.end(), id); + } + for (size_t j = found[q]; j < k; ++j) { + CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); + CATCH_REQUIRE( + results.distance(q, j) == std::numeric_limits::infinity() + ); + } + total += gt.size(); + } + return double(hits) / double(total); + }; + + auto mod10 = [](size_t e) { return e % 10 == 0; }; + + CATCH_SECTION("recall against brute force") { + CATCH_REQUIRE(check(mod10, true) >= 0.95); + } + + CATCH_SECTION("deleted ids are never returned") { + std::vector to_delete; + for (size_t i = 0; i < n; i += 3) { + to_delete.push_back(ids[i]); + } + index.delete_entries(to_delete); + deleted.insert(to_delete.begin(), to_delete.end()); + CATCH_REQUIRE(check(mod10, true) >= 0.9); + CATCH_REQUIRE(check([](size_t) { return true; }, true) >= 0.9); + } + + CATCH_SECTION("fewer than k passing ids are padded") { + auto few = [](size_t e) { return e == 100 || e == 200 || e == 300; }; + check(few, false); + auto results = svs::QueryResult(num_queries, k); + std::vector found; + index.search(results.view(), queries.cview(), sp, few, found); + for (size_t q = 0; q < num_queries; ++q) { + CATCH_REQUIRE(found[q] == 3); + } + auto none = [](size_t) { return false; }; + index.search(results.view(), queries.cview(), sp, none, found); + for (size_t q = 0; q < num_queries; ++q) { + CATCH_REQUIRE(found[q] == 0); + } + } + + CATCH_SECTION("orchestrator overload") { + auto dyn = svs::DynamicVamana::build( + parameters, + [&] { + auto d = svs::data::SimpleData(n, dim); + for (size_t i = 0; i < n; ++i) { + d.set_datum(i, raw[i]); + } + return d; + }(), + ids, + Distance(), + num_threads + ); + auto results = svs::QueryResult(num_queries, k); + std::vector found; + CATCH_REQUIRE(dyn.filtered_search( + results.view(), queries.cview(), sp, std::function(mod10), found + )); + for (size_t q = 0; q < num_queries; ++q) { + CATCH_REQUIRE(found[q] == k); + for (size_t j = 0; j < k; ++j) { + CATCH_REQUIRE(mod10(results.index(q, j))); + } + } + } +} From 8b0f4371abb15fb39fcad0655a3173212983e804 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 21 Oct 2026 14:53:14 -0700 Subject: [PATCH 02/47] Make filtered search an optional argument of the existing search Replace the separate filtered search overload with two optional trailing arguments on MutableVamanaIndex::search (an id predicate and a per-query result count). Without a predicate the existing code path runs unchanged. check_and_supplement_search_buffer gets an optional predicate that is checked before the distance is computed. The filter predicate type is now a type-erased std::function, so FilteredValidBuilder is no longer a template. The orchestrator keeps one appended virtual that forwards to the index search. --- include/svs/index/vamana/dynamic_index.h | 126 ++++++++------------- include/svs/index/vamana/extensions.h | 36 ++---- include/svs/orchestrators/dynamic_vamana.h | 22 ++-- tests/svs/index/vamana/dynamic_index.cpp | 54 ++++++++- 4 files changed, 120 insertions(+), 118 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 5c75d4445..6a0c0cefc 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -111,22 +111,17 @@ class ValidBuilder { }; // Like `ValidBuilder`, but an entry is also invalid if `pred(external_id)` is false. -template class FilteredValidBuilder { +class FilteredValidBuilder { public: FilteredValidBuilder( const std::vector& status, const IDTranslator& translator, - const Pred& pred + const extensions::IDPredicate& pred ) : status_{status} , translator_{translator} , pred_{pred} {} - template bool is_valid(I internal_id, size_t external_id) const { - return getindex(status_, internal_id) != SlotMetadata::Deleted && - pred_(external_id); - } - template PredicatedSearchNeighbor operator()(I i, float distance) const { bool valid = getindex(status_, i) != SlotMetadata::Deleted && @@ -137,7 +132,7 @@ template class FilteredValidBuilder { private: const std::vector& status_; const IDTranslator& translator_; - const Pred& pred_; + const extensions::IDPredicate& pred_; }; // Data types for which the filtered `search` overload is valid (identity accessor). @@ -673,8 +668,17 @@ class MutableVamanaIndex { QueryResultView results, const Queries& queries, const search_parameters_type& sp, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const extensions::IDPredicate* pred = nullptr, + std::vector* found = nullptr ) { + // Optional filter: only live entries with `(*pred)(external_id)` are returned; + // others are navigated through but do not count toward the search window. Rows + // with fewer than `n_neighbors` results are padded with id `size_t(-1)` and + // distance `+inf`; `found` (if given) receives the count per query. + if (found != nullptr) { + found->assign(queries.size(), 0); + } threads::parallel_for( threadpool_, threads::StaticPartition{queries.size()}, @@ -693,6 +697,39 @@ class MutableVamanaIndex { auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); + if (pred != nullptr) { + if constexpr (filtered_search_supported) { + auto builder = FilteredValidBuilder{status_, translator_, *pred}; + auto search = + greedy_search_closure(builder, prefetch_parameters, cancel); + auto accessor = data::GetDatumAccessor(); + for (auto i : is) { + auto query = queries.get_datum(i); + search(query, accessor, scratch, buffer); + extensions::check_and_supplement_search_buffer( + *this, buffer, query, pred + ); + size_t n = std::min(num_neighbors, buffer.size()); + for (size_t j = 0; j < num_neighbors; ++j) { + bool ok = j < n; + results.index(i, j) = + ok ? translate_internal_id(buffer[j].id()) : size_t(-1); + results.distance(i, j) = + ok ? buffer[j].distance() + : std::numeric_limits::infinity(); + } + if (found != nullptr) { + (*found)[i] = n; + } + } + return; + } else { + throw ANNEXCEPTION( + "Filtered search is not supported for this data type." + ); + } + } + extensions::per_thread_batch_search( data_, buffer, @@ -708,7 +745,8 @@ class MutableVamanaIndex { ); // Check if request to cancel the search - if (cancel()) { + // (Filtered results already hold external ids.) + if (cancel() || pred != nullptr) { return; } @@ -717,74 +755,6 @@ class MutableVamanaIndex { translate_to_external(results.indices()); } - /// - /// @brief Filtered top-k search: only live entries whose external id satisfies `pred` - /// are returned. Invalid entries are navigated through but do not count toward the - /// search window. - /// - /// @param pred Callable `bool(size_t external_id)`. Must be thread safe. - /// @param found Resized to `queries.size()`; receives the number of results per query. - /// - /// Rows with fewer than `n_neighbors` results are padded with id `size_t(-1)` and - /// distance `+inf`. Requires `filtered_search_supported`. - /// - template - requires(std::is_invocable_r_v) - void search( - QueryResultView results, - const Queries& queries, - const search_parameters_type& sp, - const Pred& pred, - std::vector& found - ) { - static_assert( - filtered_search_supported, - "Data type needs a custom search path (e.g. reranking)." - ); - found.assign(queries.size(), 0); - threads::parallel_for( - threadpool_, - threads::StaticPartition{queries.size()}, - [&](const auto is, uint64_t SVS_UNUSED(tid)) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - auto distance = extensions::per_thread_batch_search_setup(data_, distance_); - auto accessor = data::GetDatumAccessor(); - auto builder = FilteredValidBuilder{status_, translator_, pred}; - // The closure holds `cancel` by reference, so it must outlive the closure. - const lib::DefaultPredicate cancel = lib::Returns(lib::Const()); - auto search = greedy_search_closure(builder, prefetch_parameters, cancel); - - for (auto i : is) { - auto query = queries.get_datum(i); - search(query, accessor, distance, buffer); - extensions::check_and_supplement_search_buffer( - *this, buffer, query, builder - ); - // Everything in the buffer is valid here (cleanup() ran, and - // supplemented entries are checked before insertion). - size_t n = std::min(num_neighbors, buffer.size()); - for (size_t j = 0; j < n; ++j) { - results.index(i, j) = translate_internal_id(buffer[j].id()); - results.distance(i, j) = buffer[j].distance(); - } - for (size_t j = n; j < num_neighbors; ++j) { - results.index(i, j) = static_cast(-1); - results.distance(i, j) = std::numeric_limits::infinity(); - } - found[i] = n; - } - } - ); - } - /// /// @brief Return a unique instance of the distance function. /// diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 5b73cf7e9..edcddec2c 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -27,6 +27,9 @@ #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" +// stdlib +#include + namespace svs::index::vamana::extensions { ///// @@ -425,48 +428,33 @@ struct VamanaSingleSearchType { } }; +/// Predicate on external ids used by filtered search. +using IDPredicate = std::function; + /// In rare cases, the search buffer may not be filled with enough results. /// This can occur in dynamic indexes when many vectors have been deleted /// and the graph becomes sparsely connected. It's a corner case and should /// not happen frequently, but when it does, we may need to supplement the buffer /// with additional results. +/// If `pred` is given, entries whose external id it rejects are skipped (checked before +/// the distance is computed). template -void check_and_supplement_search_buffer( - const Index& index, SearchBuffer& search_buffer, const Query& query -) { - if (search_buffer.valid() < search_buffer.target_window() && - search_buffer.valid() < index.size()) { - search_buffer.sort(); - for (auto external_id : index.external_ids()) { - auto internal_id = index.translate_external_id(external_id); - auto dist = index.get_distance(external_id, query); - auto builder = index.internal_search_builder(); - search_buffer.insert(builder(internal_id, dist)); - if (search_buffer.valid() >= search_buffer.target_window()) { - break; - } - } - } -} - -/// Filtered variant: only ids accepted by `builder` are inserted, and the predicate is -/// evaluated before the distance is computed. Does not change the unfiltered path above. -template void check_and_supplement_search_buffer( const Index& index, SearchBuffer& search_buffer, const Query& query, - const Builder& builder + const IDPredicate* pred = nullptr ) { if (search_buffer.valid() < search_buffer.target_window() && search_buffer.valid() < index.size()) { search_buffer.sort(); for (auto external_id : index.external_ids()) { - auto internal_id = index.translate_external_id(external_id); - if (!builder.is_valid(internal_id, external_id)) { + if (pred != nullptr && !(*pred)(external_id)) { continue; } + auto internal_id = index.translate_external_id(external_id); auto dist = index.get_distance(external_id, query); + auto builder = index.internal_search_builder(); search_buffer.insert(builder(internal_id, dist)); if (search_buffer.valid() >= search_buffer.target_window()) { break; diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index e82c5c0f9..403080185 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -58,9 +58,7 @@ class DynamicVamanaInterface : public VamanaInterface { // Check if the index has been consolidated - no `Deleted` entries remain. virtual bool is_consolidated() const = 0; - // Filtered top-k on the search buffer; `filter` takes an external id. Returns false, - // leaving `results` untouched, if the data type does not support it. - // Keep last: new virtuals must be appended. + // Filtered search; false if the data type does not support it. Keep last (ABI). virtual bool filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, @@ -135,7 +133,14 @@ class DynamicVamanaImpl : public VamanaImpl(lib::Type SVS_UNUSED(type)) { const auto view = data::ConstSimpleDataView(queries); - impl().search(results, view, search_parameters, filter, found); + impl().search( + results, + view, + search_parameters, + lib::Returns(lib::Const()), + &filter, + &found + ); }, [&](svs::DataType data_type) { throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); @@ -647,14 +652,7 @@ class DynamicVamana : public manager::IndexManager { /// @brief Check if the index has been consolidated - no `Deleted` entries remain. bool is_consolidated() const { return impl_->is_consolidated(); } - /// - /// @brief Filtered top-k search on the search buffer, see - /// ``MutableVamanaIndex::search`` (filtered overload). - /// - /// @param filter Takes an external id. - /// @param found Receives the number of results per query. Rows are padded with id - /// ``size_t(-1)`` and distance ``+inf``. - /// @return False if the data type is unsupported, in which case nothing is written. + /// @brief Filtered search (see ``MutableVamanaIndex::search``); false if unsupported. template bool filtered_search( QueryResultView results, diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 7a7fadabe..c903b0470 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -508,6 +508,19 @@ CATCH_TEST_CASE( auto sp = svs::index::vamana::VamanaSearchParameters{}; sp.buffer_config_ = svs::index::vamana::SearchBufferConfig{50}; + auto search_filtered = [&](auto& results, + const svs::index::vamana::extensions::IDPredicate& pred, + std::vector& found) { + index.search( + results.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + &pred, + &found + ); + }; + // Brute force over live ids passing `pred`. std::unordered_set deleted; auto brute_force = [&](size_t q, auto&& pred) { @@ -535,7 +548,7 @@ CATCH_TEST_CASE( auto check = [&](auto&& pred, bool expect_full) { auto results = svs::QueryResult(num_queries, k); std::vector found; - index.search(results.view(), queries.cview(), sp, pred, found); + search_filtered(results, pred, found); CATCH_REQUIRE(found.size() == num_queries); size_t hits = 0, total = 0; for (size_t q = 0; q < num_queries; ++q) { @@ -583,18 +596,51 @@ CATCH_TEST_CASE( check(few, false); auto results = svs::QueryResult(num_queries, k); std::vector found; - index.search(results.view(), queries.cview(), sp, few, found); + search_filtered(results, few, found); for (size_t q = 0; q < num_queries; ++q) { CATCH_REQUIRE(found[q] == 3); } auto none = [](size_t) { return false; }; - index.search(results.view(), queries.cview(), sp, none, found); + search_filtered(results, none, found); for (size_t q = 0; q < num_queries; ++q) { CATCH_REQUIRE(found[q] == 0); } } - CATCH_SECTION("orchestrator overload") { + CATCH_SECTION("unfiltered search is unchanged by the optional arguments") { + index.delete_entries(std::vector{ids[0], ids[1], ids[2]}); + auto plain = svs::QueryResult(num_queries, k); + auto with_defaults = svs::QueryResult(num_queries, k); + auto null_args = svs::QueryResult(num_queries, k); + index.search(plain.view(), queries.cview(), sp); + index.search( + with_defaults.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + nullptr, + nullptr + ); + std::vector found; + index.search( + null_args.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + nullptr, + &found + ); + CATCH_REQUIRE(found.size() == num_queries); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 0; j < k; ++j) { + CATCH_REQUIRE(plain.index(q, j) == with_defaults.index(q, j)); + CATCH_REQUIRE(plain.index(q, j) == null_args.index(q, j)); + CATCH_REQUIRE(plain.distance(q, j) == with_defaults.distance(q, j)); + } + } + } + + CATCH_SECTION("orchestrator entry point") { auto dyn = svs::DynamicVamana::build( parameters, [&] { From a4983729c5d5055e9271f55a34ebcf5b141bdaea Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 21 Oct 2026 15:46:49 -0700 Subject: [PATCH 03/47] Run filtered search through the existing batch search path The optional filter on MutableVamanaIndex::search now only chooses which builder the existing greedy search closure uses; both cases call the same per_thread_batch_search. FilteredValidBuilder derives from ValidBuilder. The buffer top-up in single_search applies the filter when the search closure carries one. The per-query result count out-parameter is removed (callers count non-padding ids). per_thread_batch_search now pads result rows with fewer than k entries (id size_t(-1), distance +inf) instead of reading past the end of the search buffer, and the post-search id translation skips the padding. --- include/svs/index/vamana/dynamic_index.h | 106 +++++++++------------ include/svs/index/vamana/extensions.h | 14 ++- include/svs/orchestrators/dynamic_vamana.h | 14 +-- tests/svs/index/vamana/dynamic_index.cpp | 56 ++++++----- 4 files changed, 96 insertions(+), 94 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 6a0c0cefc..6fbe09c1d 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -111,30 +111,40 @@ class ValidBuilder { }; // Like `ValidBuilder`, but an entry is also invalid if `pred(external_id)` is false. -class FilteredValidBuilder { +class FilteredValidBuilder : public ValidBuilder { public: FilteredValidBuilder( const std::vector& status, const IDTranslator& translator, const extensions::IDPredicate& pred ) - : status_{status} + : ValidBuilder{status} , translator_{translator} , pred_{pred} {} template PredicatedSearchNeighbor operator()(I i, float distance) const { - bool valid = getindex(status_, i) != SlotMetadata::Deleted && - pred_(translator_.get_external(i)); - return PredicatedSearchNeighbor(i, distance, valid); + auto base = ValidBuilder::operator()(i, distance); + if (base.valid() && !pred_(translator_.get_external(i))) { + return PredicatedSearchNeighbor(i, distance, false); + } + return base; } private: - const std::vector& status_; const IDTranslator& translator_; const extensions::IDPredicate& pred_; }; +// A search closure that also carries the filter, so the buffer top-up can apply it too. +template struct FilteredSearch { + Search search; + const extensions::IDPredicate& pred; + template void operator()(Args&&... args) const { + search(std::forward(args)...); + } +}; + // Data types for which the filtered `search` overload is valid (identity accessor). template inline constexpr bool filtered_search_supported = false; template @@ -566,6 +576,9 @@ class MutableVamanaIndex { [&](const auto is, uint64_t /*tid*/) { for (auto i : is) { for (size_t j = 0, jmax = getsize<1>(ids); j < jmax; ++j) { + if (ids.at(i, j) == std::numeric_limits::max()) { + continue; // Padding from a search with fewer than k results. + } auto internal = lib::narrow_cast(ids.at(i, j)); ids.at(i, j) = translate_internal_id(internal); } @@ -669,16 +682,11 @@ class MutableVamanaIndex { const Queries& queries, const search_parameters_type& sp, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const extensions::IDPredicate* pred = nullptr, - std::vector* found = nullptr + const extensions::IDPredicate* pred = nullptr ) { - // Optional filter: only live entries with `(*pred)(external_id)` are returned; - // others are navigated through but do not count toward the search window. Rows - // with fewer than `n_neighbors` results are padded with id `size_t(-1)` and - // distance `+inf`; `found` (if given) receives the count per query. - if (found != nullptr) { - found->assign(queries.size(), 0); - } + // Optional filter (`pred` takes an external id): only live entries it accepts are + // returned. Rows with fewer than `n_neighbors` results are padded with id + // `size_t(-1)` and distance `+inf`. Only meant for `SimpleData`. threads::parallel_for( threadpool_, threads::StaticPartition{queries.size()}, @@ -697,56 +705,34 @@ class MutableVamanaIndex { auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - if (pred != nullptr) { - if constexpr (filtered_search_supported) { - auto builder = FilteredValidBuilder{status_, translator_, *pred}; - auto search = - greedy_search_closure(builder, prefetch_parameters, cancel); - auto accessor = data::GetDatumAccessor(); - for (auto i : is) { - auto query = queries.get_datum(i); - search(query, accessor, scratch, buffer); - extensions::check_and_supplement_search_buffer( - *this, buffer, query, pred - ); - size_t n = std::min(num_neighbors, buffer.size()); - for (size_t j = 0; j < num_neighbors; ++j) { - bool ok = j < n; - results.index(i, j) = - ok ? translate_internal_id(buffer[j].id()) : size_t(-1); - results.distance(i, j) = - ok ? buffer[j].distance() - : std::numeric_limits::infinity(); - } - if (found != nullptr) { - (*found)[i] = n; - } - } - return; - } else { - throw ANNEXCEPTION( - "Filtered search is not supported for this data type." - ); - } + auto run = [&](const auto& search) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); + }; + if (pred == nullptr) { + run(greedy_search_closure(prefetch_parameters, cancel)); + } else { + auto search = greedy_search_closure( + FilteredValidBuilder{status_, translator_, *pred}, + prefetch_parameters, + cancel + ); + run(FilteredSearch{search, *pred}); } - - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - greedy_search_closure(prefetch_parameters, cancel), - *this, - cancel - ); } ); // Check if request to cancel the search - // (Filtered results already hold external ids.) - if (cancel() || pred != nullptr) { + if (cancel()) { return; } diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index edcddec2c..83b47557f 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -29,6 +29,7 @@ // stdlib #include +#include namespace svs::index::vamana::extensions { @@ -495,7 +496,11 @@ SVS_FORCE_INLINE void svs_invoke( search(query, accessor, distance, search_buffer); if constexpr (Index::needs_id_translation) { - check_and_supplement_search_buffer(index, search_buffer, query); + const IDPredicate* pred = nullptr; + if constexpr (requires { search.pred; }) { + pred = &search.pred; // Filtered search: `search` carries its filter. + } + check_and_supplement_search_buffer(index, search_buffer, query, pred); } } @@ -600,7 +605,12 @@ void svs_invoke( // Copy back results. for (size_t j = 0; j < num_neighbors; ++j) { - result.set(search_buffer[j], i, j); + if (j < search_buffer.size()) { + result.set(search_buffer[j], i, j); + } else { // Fewer than k results: pad. + result.index(i, j) = static_cast(-1); + result.distance(i, j) = std::numeric_limits::infinity(); + } } } } diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 403080185..0af8f6486 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -63,8 +63,7 @@ class DynamicVamanaInterface : public VamanaInterface { svs::QueryResultView results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, - const std::function& filter, - std::vector& found + const std::function& filter ) = 0; }; @@ -124,8 +123,7 @@ class DynamicVamanaImpl : public VamanaImpl results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, - const std::function& filter, - std::vector& found + const std::function& filter ) override { if constexpr (requires { requires Impl::has_filtered_search; }) { lib::match( @@ -138,8 +136,7 @@ class DynamicVamanaImpl : public VamanaImpl()), - &filter, - &found + &filter ); }, [&](svs::DataType data_type) { @@ -658,11 +655,10 @@ class DynamicVamana : public manager::IndexManager { QueryResultView results, data::ConstSimpleDataView queries, const VamanaSearchParameters& search_parameters, - const std::function& filter, - std::vector& found + const std::function& filter ) { return impl_->filtered_search( - results, AnonymousArray<2>(queries), search_parameters, filter, found + results, AnonymousArray<2>(queries), search_parameters, filter ); } }; diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index c903b0470..0e9580f4c 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -508,6 +508,7 @@ CATCH_TEST_CASE( auto sp = svs::index::vamana::VamanaSearchParameters{}; sp.buffer_config_ = svs::index::vamana::SearchBufferConfig{50}; + // Runs the filtered search; `found` = number of non-padding ids per query. auto search_filtered = [&](auto& results, const svs::index::vamana::extensions::IDPredicate& pred, std::vector& found) { @@ -516,9 +517,14 @@ CATCH_TEST_CASE( queries.cview(), sp, svs::lib::Returns(svs::lib::Const()), - &pred, - &found + &pred ); + found.assign(num_queries, 0); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 0; j < k; ++j) { + found[q] += results.index(q, j) != static_cast(-1); + } + } }; // Brute force over live ids passing `pred`. @@ -607,35 +613,41 @@ CATCH_TEST_CASE( } } - CATCH_SECTION("unfiltered search is unchanged by the optional arguments") { + CATCH_SECTION("unfiltered search is unchanged by the optional argument") { index.delete_entries(std::vector{ids[0], ids[1], ids[2]}); auto plain = svs::QueryResult(num_queries, k); - auto with_defaults = svs::QueryResult(num_queries, k); - auto null_args = svs::QueryResult(num_queries, k); + auto with_null = svs::QueryResult(num_queries, k); index.search(plain.view(), queries.cview(), sp); index.search( - with_defaults.view(), + with_null.view(), queries.cview(), sp, svs::lib::Returns(svs::lib::Const()), - nullptr, nullptr ); - std::vector found; - index.search( - null_args.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - nullptr, - &found - ); - CATCH_REQUIRE(found.size() == num_queries); for (size_t q = 0; q < num_queries; ++q) { for (size_t j = 0; j < k; ++j) { - CATCH_REQUIRE(plain.index(q, j) == with_defaults.index(q, j)); - CATCH_REQUIRE(plain.index(q, j) == null_args.index(q, j)); - CATCH_REQUIRE(plain.distance(q, j) == with_defaults.distance(q, j)); + CATCH_REQUIRE(plain.index(q, j) == with_null.index(q, j)); + CATCH_REQUIRE(plain.distance(q, j) == with_null.distance(q, j)); + } + } + } + + CATCH_SECTION("unfiltered search pads rows with fewer than k results") { + std::vector to_delete(ids.begin() + 4, ids.end()); + index.delete_entries(to_delete); + auto results = svs::QueryResult(num_queries, k); + index.search(results.view(), queries.cview(), sp); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 0; j < k; ++j) { + if (j < 4) { + CATCH_REQUIRE(results.index(q, j) < ids[4]); + } else { + CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); + CATCH_REQUIRE( + results.distance(q, j) == std::numeric_limits::infinity() + ); + } } } } @@ -655,12 +667,10 @@ CATCH_TEST_CASE( num_threads ); auto results = svs::QueryResult(num_queries, k); - std::vector found; CATCH_REQUIRE(dyn.filtered_search( - results.view(), queries.cview(), sp, std::function(mod10), found + results.view(), queries.cview(), sp, std::function(mod10) )); for (size_t q = 0; q < num_queries; ++q) { - CATCH_REQUIRE(found[q] == k); for (size_t j = 0; j < k; ++j) { CATCH_REQUIRE(mod10(results.index(q, j))); } From 5cce383a267d76ba1a2a69a7dedf13a7361dd41d Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 21 Oct 2026 16:01:09 -0700 Subject: [PATCH 04/47] Filter through ValidBuilder in the existing dynamic search Give ValidBuilder an optional predicate and pass it down through internal_search_builder, greedy_search_closure and search() as optional arguments, instead of a separate builder and search path. Filtered-only checks: the supplement fallback skips failing ids, copy-out pads short rows with id -1 and the worst score, translation skips padded ids. --- include/svs/index/vamana/dynamic_index.h | 106 +++++++-------------- include/svs/index/vamana/extensions.h | 36 ++++--- include/svs/orchestrators/dynamic_vamana.h | 14 ++- tests/svs/index/vamana/dynamic_index.cpp | 2 +- 4 files changed, 64 insertions(+), 94 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 6fbe09c1d..a78b212e7 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -18,7 +18,6 @@ // stdlib #include -#include #include #include @@ -95,69 +94,39 @@ inline constexpr std::string_view name(SlotMetadata metadata) { class ValidBuilder { public: - ValidBuilder(const std::vector& status) - : status_{status} {} + // Optional `pred` (with `translator`): an entry is also invalid if `pred(external_id)` + // is false. Invalid entries are navigated through but never returned. + ValidBuilder( + const std::vector& status, + const IDTranslator* translator = nullptr, + const std::function* pred = nullptr + ) + : status_{status} + , translator_{translator} + , pred_{pred} {} template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. + if (!invalid && pred_ != nullptr) { + invalid = !(*pred_)(translator_->get_external(i)); + } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const std::vector& status_; + const IDTranslator* translator_; + const std::function* pred_; }; -// Like `ValidBuilder`, but an entry is also invalid if `pred(external_id)` is false. -class FilteredValidBuilder : public ValidBuilder { - public: - FilteredValidBuilder( - const std::vector& status, - const IDTranslator& translator, - const extensions::IDPredicate& pred - ) - : ValidBuilder{status} - , translator_{translator} - , pred_{pred} {} - - template - PredicatedSearchNeighbor operator()(I i, float distance) const { - auto base = ValidBuilder::operator()(i, distance); - if (base.valid() && !pred_(translator_.get_external(i))) { - return PredicatedSearchNeighbor(i, distance, false); - } - return base; - } - - private: - const IDTranslator& translator_; - const extensions::IDPredicate& pred_; -}; - -// A search closure that also carries the filter, so the buffer top-up can apply it too. -template struct FilteredSearch { - Search search; - const extensions::IDPredicate& pred; - template void operator()(Args&&... args) const { - search(std::forward(args)...); - } -}; - -// Data types for which the filtered `search` overload is valid (identity accessor). -template inline constexpr bool filtered_search_supported = false; -template -inline constexpr bool filtered_search_supported> = true; - template class MutableVamanaIndex { friend class MultiMutableVamanaIndex; public: - // Present only on indexes that support the filtered `search` overload. - static constexpr bool has_filtered_search = filtered_search_supported; - // Traits static constexpr bool supports_insertions = true; static constexpr bool supports_deletions = true; @@ -576,8 +545,8 @@ class MutableVamanaIndex { [&](const auto is, uint64_t /*tid*/) { for (auto i : is) { for (size_t j = 0, jmax = getsize<1>(ids); j < jmax; ++j) { - if (ids.at(i, j) == std::numeric_limits::max()) { - continue; // Padding from a search with fewer than k results. + if (ids.at(i, j) == static_cast(-1)) { + continue; // Padding from a filtered search. } auto internal = lib::narrow_cast(ids.at(i, j)); ids.at(i, j) = translate_internal_id(internal); @@ -620,15 +589,17 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + ValidBuilder internal_search_builder(const std::function* pred = nullptr + ) const { + return ValidBuilder{status_, &translator_, pred}; + } - template auto greedy_search_closure( - const Builder& builder, GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* pred = nullptr ) const { - return [&, builder, prefetch_parameters]( + return [&, prefetch_parameters, pred]( const auto& query, auto& accessor, auto& distance, auto& buffer ) { // Perform the greedy search using the provided resources. @@ -640,7 +611,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - builder, + internal_search_builder(pred), prefetch_parameters, cancel ); @@ -650,15 +621,6 @@ class MutableVamanaIndex { }; } - auto greedy_search_closure( - GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) - ) const { - return greedy_search_closure( - internal_search_builder(), prefetch_parameters, cancel - ); - } - // Single Search template void search( @@ -682,11 +644,11 @@ class MutableVamanaIndex { const Queries& queries, const search_parameters_type& sp, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const extensions::IDPredicate* pred = nullptr + const std::function* pred = nullptr ) { - // Optional filter (`pred` takes an external id): only live entries it accepts are - // returned. Rows with fewer than `n_neighbors` results are padded with id - // `size_t(-1)` and distance `+inf`. Only meant for `SimpleData`. + // Optional `pred`: only entries whose external id passes are returned. It is called + // from all search threads. Rows with fewer passing entries than requested are + // padded with id `size_t(-1)` and the worst score. threads::parallel_for( threadpool_, threads::StaticPartition{queries.size()}, @@ -718,15 +680,11 @@ class MutableVamanaIndex { cancel ); }; + auto search = greedy_search_closure(prefetch_parameters, cancel, pred); if (pred == nullptr) { - run(greedy_search_closure(prefetch_parameters, cancel)); + run(search); } else { - auto search = greedy_search_closure( - FilteredValidBuilder{status_, translator_, *pred}, - prefetch_parameters, - cancel - ); - run(FilteredSearch{search, *pred}); + run(extensions::FilteredSearch{search, pred}); } } ); diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 83b47557f..883286b27 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -27,7 +27,6 @@ #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" -// stdlib #include #include @@ -429,22 +428,17 @@ struct VamanaSingleSearchType { } }; -/// Predicate on external ids used by filtered search. -using IDPredicate = std::function; - /// In rare cases, the search buffer may not be filled with enough results. /// This can occur in dynamic indexes when many vectors have been deleted /// and the graph becomes sparsely connected. It's a corner case and should /// not happen frequently, but when it does, we may need to supplement the buffer /// with additional results. -/// If `pred` is given, entries whose external id it rejects are skipped (checked before -/// the distance is computed). template void check_and_supplement_search_buffer( const Index& index, SearchBuffer& search_buffer, const Query& query, - const IDPredicate* pred = nullptr + const std::function* pred = nullptr ) { if (search_buffer.valid() < search_buffer.target_window() && search_buffer.valid() < index.size()) { @@ -464,6 +458,15 @@ void check_and_supplement_search_buffer( } } +/// A search closure that also carries the filter of a filtered search. +template struct FilteredSearch { + Search search; + const std::function* pred; + template void operator()(Args&&... args) const { + search(std::forward(args)...); + } +}; + /// Customization point object for processing single queries. inline constexpr VamanaSingleSearchType single_search{}; @@ -496,9 +499,9 @@ SVS_FORCE_INLINE void svs_invoke( search(query, accessor, distance, search_buffer); if constexpr (Index::needs_id_translation) { - const IDPredicate* pred = nullptr; + const std::function* pred = nullptr; if constexpr (requires { search.pred; }) { - pred = &search.pred; // Filtered search: `search` carries its filter. + pred = search.pred; // Filtered search: keep filtered-out ids out of the top-up. } check_and_supplement_search_buffer(index, search_buffer, query, pred); } @@ -605,12 +608,17 @@ void svs_invoke( // Copy back results. for (size_t j = 0; j < num_neighbors; ++j) { - if (j < search_buffer.size()) { - result.set(search_buffer[j], i, j); - } else { // Fewer than k results: pad. - result.index(i, j) = static_cast(-1); - result.distance(i, j) = std::numeric_limits::infinity(); + if constexpr (requires { search.pred; }) { + if (j >= search_buffer.size()) { // Filtered: fewer than k passed. + result.index(i, j) = static_cast(-1); + result.distance(i, j) = + distance::comparator(distance)(0.0f, 1.0f) + ? std::numeric_limits::infinity() + : -std::numeric_limits::infinity(); + continue; + } } + result.set(search_buffer[j], i, j); } } } diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 0af8f6486..da4e01137 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -22,7 +22,6 @@ #include "svs/orchestrators/vamana.h" // stdlib -#include #include namespace svs { @@ -67,6 +66,11 @@ class DynamicVamanaInterface : public VamanaInterface { ) = 0; }; +// Filtered search is only enabled for uncompressed data. +template inline constexpr bool is_simple_data = false; +template +inline constexpr bool is_simple_data> = true; + template class DynamicVamanaImpl : public VamanaImpl { public: @@ -125,15 +129,14 @@ class DynamicVamanaImpl : public VamanaImpl& filter ) override { - if constexpr (requires { requires Impl::has_filtered_search; }) { + if constexpr (is_simple_data) { lib::match( QueryTypes{}, queries.type(), [&](lib::Type SVS_UNUSED(type)) { - const auto view = data::ConstSimpleDataView(queries); impl().search( results, - view, + data::ConstSimpleDataView(queries), search_parameters, lib::Returns(lib::Const()), &filter @@ -649,7 +652,8 @@ class DynamicVamana : public manager::IndexManager { /// @brief Check if the index has been consolidated - no `Deleted` entries remain. bool is_consolidated() const { return impl_->is_consolidated(); } - /// @brief Filtered search (see ``MutableVamanaIndex::search``); false if unsupported. + /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. + /// Returns false if the data type does not support it. template bool filtered_search( QueryResultView results, diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 0e9580f4c..50fe5b282 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -510,7 +510,7 @@ CATCH_TEST_CASE( // Runs the filtered search; `found` = number of non-padding ids per query. auto search_filtered = [&](auto& results, - const svs::index::vamana::extensions::IDPredicate& pred, + const std::function& pred, std::vector& found) { index.search( results.view(), From c4d7396e54b4689b43e9b4a15dfc417d8d3a1c85 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 21 Oct 2026 22:12:04 -0700 Subject: [PATCH 05/47] Fix filtered test and orchestrator for the concurrent index The orchestrator reports filtered search as unsupported for indexes whose search has no filter argument (the concurrent index), instead of failing to compile. In the filtered test, drop the unfiltered-padding section (padding is filtered-only) and add an inner-product case checking padding uses -inf. --- include/svs/orchestrators/dynamic_vamana.h | 13 +++++++- tests/svs/index/vamana/dynamic_index.cpp | 35 ++++++++++++++-------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index da4e01137..9619bdc7d 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -129,7 +129,18 @@ class DynamicVamanaImpl : public VamanaImpl& filter ) override { - if constexpr (is_simple_data) { + // Indexes whose `search` has no filter argument (e.g. the concurrent index) report + // "unsupported" like other data types. + constexpr bool supported = + is_simple_data && + requires( + Impl & idx, + svs::QueryResultView r, + data::ConstSimpleDataView q, + const index::vamana::VamanaSearchParameters& sp, + const std::function* pred + ) { idx.search(r, q, sp, lib::Returns(lib::Const()), pred); }; + if constexpr (supported) { lib::match( QueryTypes{}, queries.type(), diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 50fe5b282..44ed215d7 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -633,21 +633,30 @@ CATCH_TEST_CASE( } } - CATCH_SECTION("unfiltered search pads rows with fewer than k results") { - std::vector to_delete(ids.begin() + 4, ids.end()); - index.delete_entries(to_delete); + CATCH_SECTION("inner product pads with the worst score (-inf)") { + auto data_ip = svs::data::BlockedData(n, dim); + for (size_t i = 0; i < n; ++i) { + data_ip.set_datum(i, raw[i]); + } + svs::index::vamana::VamanaBuildParameters parameters_ip{0.95, 32, 64, 64, 32, true}; + auto index_ip = svs::index::vamana::MutableVamanaIndex( + parameters_ip, std::move(data_ip), ids, svs::distance::DistanceIP(), num_threads + ); + std::function few = [](size_t e) { return e == 100 || e == 200; }; auto results = svs::QueryResult(num_queries, k); - index.search(results.view(), queries.cview(), sp); + index_ip.search( + results.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + &few + ); for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - if (j < 4) { - CATCH_REQUIRE(results.index(q, j) < ids[4]); - } else { - CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); - CATCH_REQUIRE( - results.distance(q, j) == std::numeric_limits::infinity() - ); - } + for (size_t j = 2; j < k; ++j) { + CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); + CATCH_REQUIRE( + results.distance(q, j) == -std::numeric_limits::infinity() + ); } } } From 1ecd4e8e768d291e4b1e98ef400e397ed2388d2d Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 26 Oct 2026 23:27:28 -0700 Subject: [PATCH 06/47] Filtered search: skip the filter for entries the buffer would drop ValidBuilder can be given the search buffer. With a filter, it first asks the buffer whether the entry would be dropped anyway (can_skip) and, if so, returns it as invalid without looking up the external id or calling the filter. Results are unchanged. --- include/svs/index/vamana/dynamic_index.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index a78b212e7..11c60fc36 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -105,8 +105,19 @@ class ValidBuilder { , translator_{translator} , pred_{pred} {} + /// With a predicate: skip the predicate for entries `buffer` would drop anyway. + template void skip_if_dropped_by(const Buffer& buffer) { + skip_buffer_ = &buffer; + skip_ = [](const void* b, float d) { + return static_cast(b)->can_skip(d); + }; + } + template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + if (pred_ != nullptr && skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); + } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. @@ -120,6 +131,8 @@ class ValidBuilder { const std::vector& status_; const IDTranslator* translator_; const std::function* pred_; + const void* skip_buffer_ = nullptr; + bool (*skip_)(const void*, float) = nullptr; }; template @@ -602,6 +615,10 @@ class MutableVamanaIndex { return [&, prefetch_parameters, pred]( const auto& query, auto& accessor, auto& distance, auto& buffer ) { + auto builder = internal_search_builder(pred); + if constexpr (requires { buffer.can_skip(0.0f); }) { + builder.skip_if_dropped_by(buffer); + } // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -611,7 +628,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(pred), + builder, prefetch_parameters, cancel ); From 6593e8572940b9c215ebc8d8682d8c1cc692963c Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Tue, 27 Oct 2026 10:48:43 -0700 Subject: [PATCH 07/47] Revert "Filtered search: skip the filter for entries the buffer would drop" This reverts commit 1ecd4e8e768d291e4b1e98ef400e397ed2388d2d. --- include/svs/index/vamana/dynamic_index.h | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 11c60fc36..a78b212e7 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -105,19 +105,8 @@ class ValidBuilder { , translator_{translator} , pred_{pred} {} - /// With a predicate: skip the predicate for entries `buffer` would drop anyway. - template void skip_if_dropped_by(const Buffer& buffer) { - skip_buffer_ = &buffer; - skip_ = [](const void* b, float d) { - return static_cast(b)->can_skip(d); - }; - } - template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if (pred_ != nullptr && skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. @@ -131,8 +120,6 @@ class ValidBuilder { const std::vector& status_; const IDTranslator* translator_; const std::function* pred_; - const void* skip_buffer_ = nullptr; - bool (*skip_)(const void*, float) = nullptr; }; template @@ -615,10 +602,6 @@ class MutableVamanaIndex { return [&, prefetch_parameters, pred]( const auto& query, auto& accessor, auto& distance, auto& buffer ) { - auto builder = internal_search_builder(pred); - if constexpr (requires { buffer.can_skip(0.0f); }) { - builder.skip_if_dropped_by(buffer); - } // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -628,7 +611,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - builder, + internal_search_builder(pred), prefetch_parameters, cancel ); From 4c42304f490cb1a07b96d8b7efa41c4f758b3c72 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 28 Oct 2026 16:21:39 -0700 Subject: [PATCH 08/47] Allow filtered search for data types that opt in --- include/svs/index/vamana/extensions.h | 17 +++++++++++++++++ include/svs/orchestrators/dynamic_vamana.h | 7 +------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 883286b27..c63e453d7 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -639,6 +639,23 @@ template struct UsesReranking { template inline constexpr UsesReranking calibration_uses_reranking{}; +/// Whether `DynamicVamana::filtered_search` is supported for `Data`. +/// Data types opt in by providing an `svs_invoke` overload. +template struct SupportsFilteredSearch { + constexpr bool operator()() const { + if constexpr (svs::svs_invocable) { + return svs::svs_invoke(*this); + } else { + return false; + } + } +}; + +template +constexpr bool svs_invoke(SupportsFilteredSearch>) { + return true; +} + ///// ///// Reconstruct Vector ///// diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 9619bdc7d..4e78ac340 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -66,11 +66,6 @@ class DynamicVamanaInterface : public VamanaInterface { ) = 0; }; -// Filtered search is only enabled for uncompressed data. -template inline constexpr bool is_simple_data = false; -template -inline constexpr bool is_simple_data> = true; - template class DynamicVamanaImpl : public VamanaImpl { public: @@ -132,7 +127,7 @@ class DynamicVamanaImpl : public VamanaImpl && + index::vamana::extensions::SupportsFilteredSearch{}() && requires( Impl & idx, svs::QueryResultView r, From c7883927397f8b1a18ddfd4cb5d54a907a3ac18e Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 29 Oct 2026 14:30:06 -0700 Subject: [PATCH 09/47] Apply clang-format --- include/svs/index/vamana/extensions.h | 7 +++---- include/svs/orchestrators/dynamic_vamana.h | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index c63e453d7..2bf8c38e2 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -611,10 +611,9 @@ void svs_invoke( if constexpr (requires { search.pred; }) { if (j >= search_buffer.size()) { // Filtered: fewer than k passed. result.index(i, j) = static_cast(-1); - result.distance(i, j) = - distance::comparator(distance)(0.0f, 1.0f) - ? std::numeric_limits::infinity() - : -std::numeric_limits::infinity(); + result.distance(i, j) = distance::comparator(distance)(0.0f, 1.0f) + ? std::numeric_limits::infinity() + : -std::numeric_limits::infinity(); continue; } } diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 4e78ac340..24f3c7f8d 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -127,7 +127,8 @@ class DynamicVamanaImpl : public VamanaImpl{}() && + index::vamana::extensions::SupportsFilteredSearch{}( + ) && requires( Impl & idx, svs::QueryResultView r, From 682ae1fa9b51e817798f508745c0397891b042d0 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 30 Oct 2026 11:20:15 -0700 Subject: [PATCH 10/47] Enable filtered search for all data types --- include/svs/index/vamana/extensions.h | 17 ----------------- include/svs/orchestrators/dynamic_vamana.h | 6 ++---- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 2bf8c38e2..74c347685 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -638,23 +638,6 @@ template struct UsesReranking { template inline constexpr UsesReranking calibration_uses_reranking{}; -/// Whether `DynamicVamana::filtered_search` is supported for `Data`. -/// Data types opt in by providing an `svs_invoke` overload. -template struct SupportsFilteredSearch { - constexpr bool operator()() const { - if constexpr (svs::svs_invocable) { - return svs::svs_invoke(*this); - } else { - return false; - } - } -}; - -template -constexpr bool svs_invoke(SupportsFilteredSearch>) { - return true; -} - ///// ///// Reconstruct Vector ///// diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 24f3c7f8d..7d5ef3e36 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -124,11 +124,9 @@ class DynamicVamanaImpl : public VamanaImpl& filter ) override { - // Indexes whose `search` has no filter argument (e.g. the concurrent index) report - // "unsupported" like other data types. + // Filtered search is available for every data type. Indexes whose `search` has no + // filter argument (e.g. the concurrent index) report "unsupported". constexpr bool supported = - index::vamana::extensions::SupportsFilteredSearch{}( - ) && requires( Impl & idx, svs::QueryResultView r, From 2f20b70e7ff862fa96ff20e79a6c1d566ede5e2c Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 30 Oct 2026 11:22:26 -0700 Subject: [PATCH 11/47] Apply clang-format --- include/svs/orchestrators/dynamic_vamana.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 7d5ef3e36..bc01a2828 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -126,14 +126,13 @@ class DynamicVamanaImpl : public VamanaImpl r, - data::ConstSimpleDataView q, - const index::vamana::VamanaSearchParameters& sp, - const std::function* pred - ) { idx.search(r, q, sp, lib::Returns(lib::Const()), pred); }; + constexpr bool supported = requires( + Impl & idx, + svs::QueryResultView r, + data::ConstSimpleDataView q, + const index::vamana::VamanaSearchParameters& sp, + const std::function* pred + ) { idx.search(r, q, sp, lib::Returns(lib::Const()), pred); }; if constexpr (supported) { lib::match( QueryTypes{}, From 94b22debcf09653b8a5f4e158f3455e931ee15ea Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 30 Oct 2026 13:35:10 -0700 Subject: [PATCH 12/47] Name the filter argument `filter` everywhere --- include/svs/index/vamana/dynamic_index.h | 32 +++++++++++----------- include/svs/index/vamana/extensions.h | 16 +++++------ include/svs/orchestrators/dynamic_vamana.h | 4 +-- tests/svs/index/vamana/dynamic_index.cpp | 18 ++++++------ 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index a78b212e7..38bf55194 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -94,24 +94,24 @@ inline constexpr std::string_view name(SlotMetadata metadata) { class ValidBuilder { public: - // Optional `pred` (with `translator`): an entry is also invalid if `pred(external_id)` + // Optional `filter` (with `translator`): an entry is also invalid if `filter(external_id)` // is false. Invalid entries are navigated through but never returned. ValidBuilder( const std::vector& status, const IDTranslator* translator = nullptr, - const std::function* pred = nullptr + const std::function* filter = nullptr ) : status_{status} , translator_{translator} - , pred_{pred} {} + , filter_{filter} {} template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. - if (!invalid && pred_ != nullptr) { - invalid = !(*pred_)(translator_->get_external(i)); + if (!invalid && filter_ != nullptr) { + invalid = !(*filter_)(translator_->get_external(i)); } return PredicatedSearchNeighbor(i, distance, !invalid); } @@ -119,7 +119,7 @@ class ValidBuilder { private: const std::vector& status_; const IDTranslator* translator_; - const std::function* pred_; + const std::function* filter_; }; template @@ -589,17 +589,17 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder(const std::function* pred = nullptr + ValidBuilder internal_search_builder(const std::function* filter = nullptr ) const { - return ValidBuilder{status_, &translator_, pred}; + return ValidBuilder{status_, &translator_, filter}; } auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* pred = nullptr + const std::function* filter = nullptr ) const { - return [&, prefetch_parameters, pred]( + return [&, prefetch_parameters, filter]( const auto& query, auto& accessor, auto& distance, auto& buffer ) { // Perform the greedy search using the provided resources. @@ -611,7 +611,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(pred), + internal_search_builder(filter), prefetch_parameters, cancel ); @@ -644,9 +644,9 @@ class MutableVamanaIndex { const Queries& queries, const search_parameters_type& sp, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* pred = nullptr + const std::function* filter = nullptr ) { - // Optional `pred`: only entries whose external id passes are returned. It is called + // Optional `filter`: only entries whose external id passes are returned. It is called // from all search threads. Rows with fewer passing entries than requested are // padded with id `size_t(-1)` and the worst score. threads::parallel_for( @@ -680,11 +680,11 @@ class MutableVamanaIndex { cancel ); }; - auto search = greedy_search_closure(prefetch_parameters, cancel, pred); - if (pred == nullptr) { + auto search = greedy_search_closure(prefetch_parameters, cancel, filter); + if (filter == nullptr) { run(search); } else { - run(extensions::FilteredSearch{search, pred}); + run(extensions::FilteredSearch{search, filter}); } } ); diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 74c347685..75e352809 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -438,13 +438,13 @@ void check_and_supplement_search_buffer( const Index& index, SearchBuffer& search_buffer, const Query& query, - const std::function* pred = nullptr + const std::function* filter = nullptr ) { if (search_buffer.valid() < search_buffer.target_window() && search_buffer.valid() < index.size()) { search_buffer.sort(); for (auto external_id : index.external_ids()) { - if (pred != nullptr && !(*pred)(external_id)) { + if (filter != nullptr && !(*filter)(external_id)) { continue; } auto internal_id = index.translate_external_id(external_id); @@ -461,7 +461,7 @@ void check_and_supplement_search_buffer( /// A search closure that also carries the filter of a filtered search. template struct FilteredSearch { Search search; - const std::function* pred; + const std::function* filter; template void operator()(Args&&... args) const { search(std::forward(args)...); } @@ -499,11 +499,11 @@ SVS_FORCE_INLINE void svs_invoke( search(query, accessor, distance, search_buffer); if constexpr (Index::needs_id_translation) { - const std::function* pred = nullptr; - if constexpr (requires { search.pred; }) { - pred = search.pred; // Filtered search: keep filtered-out ids out of the top-up. + const std::function* filter = nullptr; + if constexpr (requires { search.filter; }) { + filter = search.filter; // Filtered search: keep filtered-out ids out of the top-up. } - check_and_supplement_search_buffer(index, search_buffer, query, pred); + check_and_supplement_search_buffer(index, search_buffer, query, filter); } } @@ -608,7 +608,7 @@ void svs_invoke( // Copy back results. for (size_t j = 0; j < num_neighbors; ++j) { - if constexpr (requires { search.pred; }) { + if constexpr (requires { search.filter; }) { if (j >= search_buffer.size()) { // Filtered: fewer than k passed. result.index(i, j) = static_cast(-1); result.distance(i, j) = distance::comparator(distance)(0.0f, 1.0f) diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index bc01a2828..1fb30f041 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -131,8 +131,8 @@ class DynamicVamanaImpl : public VamanaImpl r, data::ConstSimpleDataView q, const index::vamana::VamanaSearchParameters& sp, - const std::function* pred - ) { idx.search(r, q, sp, lib::Returns(lib::Const()), pred); }; + const std::function* filter + ) { idx.search(r, q, sp, lib::Returns(lib::Const()), filter); }; if constexpr (supported) { lib::match( QueryTypes{}, diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 44ed215d7..6bedb8ea8 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -510,14 +510,14 @@ CATCH_TEST_CASE( // Runs the filtered search; `found` = number of non-padding ids per query. auto search_filtered = [&](auto& results, - const std::function& pred, + const std::function& filter, std::vector& found) { index.search( results.view(), queries.cview(), sp, svs::lib::Returns(svs::lib::Const()), - &pred + &filter ); found.assign(num_queries, 0); for (size_t q = 0; q < num_queries; ++q) { @@ -527,13 +527,13 @@ CATCH_TEST_CASE( } }; - // Brute force over live ids passing `pred`. + // Brute force over live ids passing `filter`. std::unordered_set deleted; - auto brute_force = [&](size_t q, auto&& pred) { + auto brute_force = [&](size_t q, auto&& filter) { std::vector> all; for (size_t i = 0; i < n; ++i) { size_t e = ids[i]; - if (deleted.contains(e) || !pred(e)) { + if (deleted.contains(e) || !filter(e)) { continue; } float d = 0; @@ -551,21 +551,21 @@ CATCH_TEST_CASE( return out; }; - auto check = [&](auto&& pred, bool expect_full) { + auto check = [&](auto&& filter, bool expect_full) { auto results = svs::QueryResult(num_queries, k); std::vector found; - search_filtered(results, pred, found); + search_filtered(results, filter, found); CATCH_REQUIRE(found.size() == num_queries); size_t hits = 0, total = 0; for (size_t q = 0; q < num_queries; ++q) { - auto gt = brute_force(q, pred); + auto gt = brute_force(q, filter); CATCH_REQUIRE(found[q] <= k); if (expect_full) { CATCH_REQUIRE(found[q] == k); } for (size_t j = 0; j < found[q]; ++j) { size_t id = results.index(q, j); - CATCH_REQUIRE(pred(id)); + CATCH_REQUIRE(filter(id)); CATCH_REQUIRE(!deleted.contains(id)); hits += std::count(gt.begin(), gt.end(), id); } From 9f8da25465250adaf8cdb1361e5601e6ed9e301b Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 30 Oct 2026 13:35:19 -0700 Subject: [PATCH 13/47] Apply clang-format --- include/svs/index/vamana/dynamic_index.h | 21 +++++++++++---------- include/svs/index/vamana/extensions.h | 3 ++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 38bf55194..f93f5c8dc 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -94,8 +94,9 @@ inline constexpr std::string_view name(SlotMetadata metadata) { class ValidBuilder { public: - // Optional `filter` (with `translator`): an entry is also invalid if `filter(external_id)` - // is false. Invalid entries are navigated through but never returned. + // Optional `filter` (with `translator`): an entry is also invalid if + // `filter(external_id)` is false. Invalid entries are navigated through but never + // returned. ValidBuilder( const std::vector& status, const IDTranslator* translator = nullptr, @@ -589,8 +590,8 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder(const std::function* filter = nullptr - ) const { + ValidBuilder + internal_search_builder(const std::function* filter = nullptr) const { return ValidBuilder{status_, &translator_, filter}; } @@ -599,9 +600,9 @@ class MutableVamanaIndex { const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - return [&, prefetch_parameters, filter]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { + return [&, + prefetch_parameters, + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -646,9 +647,9 @@ class MutableVamanaIndex { const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) { - // Optional `filter`: only entries whose external id passes are returned. It is called - // from all search threads. Rows with fewer passing entries than requested are - // padded with id `size_t(-1)` and the worst score. + // Optional `filter`: only entries whose external id passes are returned. It is + // called from all search threads. Rows with fewer passing entries than requested + // are padded with id `size_t(-1)` and the worst score. threads::parallel_for( threadpool_, threads::StaticPartition{queries.size()}, diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 75e352809..9ec594486 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -501,7 +501,8 @@ SVS_FORCE_INLINE void svs_invoke( if constexpr (Index::needs_id_translation) { const std::function* filter = nullptr; if constexpr (requires { search.filter; }) { - filter = search.filter; // Filtered search: keep filtered-out ids out of the top-up. + filter = + search.filter; // Filtered search: keep filtered-out ids out of the top-up. } check_and_supplement_search_buffer(index, search_buffer, query, filter); } From a389cf8d89bf403859e5d4e9a5c1eb4a3d1975c9 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 30 Oct 2026 23:26:01 -0700 Subject: [PATCH 14/47] Keep the unfiltered search path inlined --- include/svs/index/vamana/dynamic_index.h | 6 +++--- include/svs/index/vamana/greedy_search.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index f93f5c8dc..219fbe0f7 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -600,9 +600,9 @@ class MutableVamanaIndex { const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - return [&, - prefetch_parameters, - filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + return [&, prefetch_parameters, filter] [[gnu::always_inline]] ( + const auto& query, auto& accessor, auto& distance, auto& buffer + ) { // Perform the greedy search using the provided resources. greedy_search( graph_, diff --git a/include/svs/index/vamana/greedy_search.h b/include/svs/index/vamana/greedy_search.h index f12c0129f..714315eb2 100644 --- a/include/svs/index/vamana/greedy_search.h +++ b/include/svs/index/vamana/greedy_search.h @@ -121,7 +121,7 @@ template < typename Initializer, typename Builder, GreedySearchTracker Tracker> -void greedy_search( +SVS_FORCE_INLINE void greedy_search( const Graph& graph, const Dataset& dataset, Accessor& accessor, @@ -213,7 +213,7 @@ template < typename Buffer, typename Initializer, typename Builder = NeighborBuilder> -void greedy_search( +SVS_FORCE_INLINE void greedy_search( const Graph& graph, const Dataset& dataset, Accessor& accessor, From f42507a121df4e7f705f50b4cde583c810c6ee8a Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 2 Nov 2026 11:14:13 -0800 Subject: [PATCH 15/47] Revert "Keep the unfiltered search path inlined" This reverts commit a389cf8d89bf403859e5d4e9a5c1eb4a3d1975c9. --- include/svs/index/vamana/dynamic_index.h | 6 +++--- include/svs/index/vamana/greedy_search.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 219fbe0f7..f93f5c8dc 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -600,9 +600,9 @@ class MutableVamanaIndex { const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - return [&, prefetch_parameters, filter] [[gnu::always_inline]] ( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { + return [&, + prefetch_parameters, + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { // Perform the greedy search using the provided resources. greedy_search( graph_, diff --git a/include/svs/index/vamana/greedy_search.h b/include/svs/index/vamana/greedy_search.h index 714315eb2..f12c0129f 100644 --- a/include/svs/index/vamana/greedy_search.h +++ b/include/svs/index/vamana/greedy_search.h @@ -121,7 +121,7 @@ template < typename Initializer, typename Builder, GreedySearchTracker Tracker> -SVS_FORCE_INLINE void greedy_search( +void greedy_search( const Graph& graph, const Dataset& dataset, Accessor& accessor, @@ -213,7 +213,7 @@ template < typename Buffer, typename Initializer, typename Builder = NeighborBuilder> -SVS_FORCE_INLINE void greedy_search( +void greedy_search( const Graph& graph, const Dataset& dataset, Accessor& accessor, From bc0a1cf1465c8043f87d3f2f16d87d6a2441a5a6 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 2 Nov 2026 11:16:42 -0800 Subject: [PATCH 16/47] Choose the filtering builder at compile time --- include/svs/index/vamana/dynamic_index.h | 153 ++++++++++++++++------- 1 file changed, 111 insertions(+), 42 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index f93f5c8dc..86215e7cf 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -94,13 +94,29 @@ inline constexpr std::string_view name(SlotMetadata metadata) { class ValidBuilder { public: - // Optional `filter` (with `translator`): an entry is also invalid if - // `filter(external_id)` is false. Invalid entries are navigated through but never - // returned. - ValidBuilder( + ValidBuilder(const std::vector& status) + : status_{status} {} + + template + constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + bool invalid = getindex(status_, i) == SlotMetadata::Deleted; + // This neighbor should be skipped if the metadata corresponding to the given index + // marks this slot as deleted. + return PredicatedSearchNeighbor(i, distance, !invalid); + } + + private: + const std::vector& status_; +}; + +// Like `ValidBuilder`, but an entry is also invalid if `filter(external_id)` is false. +// Invalid entries are navigated through but never returned. Used by filtered searches. +class FilteringValidBuilder { + public: + FilteringValidBuilder( const std::vector& status, - const IDTranslator* translator = nullptr, - const std::function* filter = nullptr + const IDTranslator& translator, + const std::function& filter ) : status_{status} , translator_{translator} @@ -108,19 +124,15 @@ class ValidBuilder { template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - bool invalid = getindex(status_, i) == SlotMetadata::Deleted; - // This neighbor should be skipped if the metadata corresponding to the given index - // marks this slot as deleted. - if (!invalid && filter_ != nullptr) { - invalid = !(*filter_)(translator_->get_external(i)); - } + bool invalid = getindex(status_, i) == SlotMetadata::Deleted || + !filter_(translator_.get_external(i)); return PredicatedSearchNeighbor(i, distance, !invalid); } private: const std::vector& status_; - const IDTranslator* translator_; - const std::function* filter_; + const IDTranslator& translator_; + const std::function& filter_; }; template @@ -590,19 +602,42 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder - internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &translator_, filter}; - } + ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* filter = nullptr + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) const { - return [&, + return [&, prefetch_parameters]( + const auto& query, auto& accessor, auto& distance, auto& buffer + ) { + // Perform the greedy search using the provided resources. + greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + internal_search_builder(), prefetch_parameters, - filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + cancel + ); + // Take a pass over the search buffer to remove any deleted elements that + // might remain. + buffer.cleanup(); + }; + } + + auto filtered_greedy_search_closure( + GreedySearchPrefetchParameters prefetch_parameters, + const lib::DefaultPredicate& cancel, + const std::function& filter + ) const { + return [&, prefetch_parameters]( + const auto& query, auto& accessor, auto& distance, auto& buffer + ) { // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -612,7 +647,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(filter), + FilteringValidBuilder{status_, translator_, filter}, prefetch_parameters, cancel ); @@ -654,6 +689,10 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { + if (filter != nullptr) { + filtered_search_thread(results, queries, sp, cancel, *filter, is); + return; + } size_t num_neighbors = results.n_neighbors(); auto buffer = search_buffer_type{ // Legalize search buffer for this search. @@ -668,25 +707,17 @@ class MutableVamanaIndex { auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - auto run = [&](const auto& search) { - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; - auto search = greedy_search_closure(prefetch_parameters, cancel, filter); - if (filter == nullptr) { - run(search); - } else { - run(extensions::FilteredSearch{search, filter}); - } + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure(prefetch_parameters, cancel), + *this, + cancel + ); } ); @@ -700,6 +731,44 @@ class MutableVamanaIndex { translate_to_external(results.indices()); } + private: + // The part of `search` that handles a filter. Kept apart so that the unfiltered search + // above is compiled with the regular builder only. + template + SVS_NOINLINE void filtered_search_thread( + QueryResultView& results, + const Queries& queries, + const search_parameters_type& sp, + const lib::DefaultPredicate& cancel, + const std::function& filter, + const Range& is + ) { + size_t num_neighbors = results.n_neighbors(); + // Legalize search buffer for this search. + auto config = sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_; + auto buffer = search_buffer_type{ + config, distance::comparator(distance_), sp.search_buffer_visited_set_}; + + auto prefetch_parameters = + GreedySearchPrefetchParameters{sp.prefetch_lookahead_, sp.prefetch_step_}; + auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); + auto search = filtered_greedy_search_closure(prefetch_parameters, cancel, filter); + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + extensions::FilteredSearch{search, &filter}, + *this, + cancel + ); + } + + public: /// /// @brief Return a unique instance of the distance function. /// From dcfed41f4a4d065de692e2e2af40af449c26ed6b Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 2 Nov 2026 13:19:54 -0800 Subject: [PATCH 17/47] Remove the added filtered-search test --- tests/svs/index/vamana/dynamic_index.cpp | 222 ----------------------- 1 file changed, 222 deletions(-) diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 6bedb8ea8..94daddcf1 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -23,14 +23,11 @@ #include #include #include -#include #include #include -#include #include #include #include -#include // svs #include "svs/core/recall.h" @@ -467,222 +464,3 @@ CATCH_TEST_CASE("MutableVamana Index Memory Usage", "[graph_index][dynamic_index const size_t usage = index.get_memory_breakdown().total(); CATCH_REQUIRE(usage == expected_total_bytes); } - -CATCH_TEST_CASE( - "MutableVamana Index Filtered Search", "[graph_index][dynamic_index][filtered]" -) { - const size_t num_threads = 2; - const size_t n = 3000; - const size_t dim = 32; - const size_t num_queries = 50; - const size_t k = 10; - using Distance = svs::distance::DistanceL2; - - std::mt19937 rng(42); - std::normal_distribution normal; - auto data = svs::data::BlockedData(n, dim); - std::vector> raw(n, std::vector(dim)); - for (size_t i = 0; i < n; ++i) { - for (auto& x : raw[i]) { - x = normal(rng); - } - data.set_datum(i, raw[i]); - } - auto queries = svs::data::SimpleData(num_queries, dim); - for (size_t i = 0; i < num_queries; ++i) { - std::vector q(dim); - for (auto& x : q) { - x = normal(rng); - } - queries.set_datum(i, q); - } - - // External ids differ from internal ids. - std::vector ids(n); - std::iota(ids.begin(), ids.end(), 7); - svs::index::vamana::VamanaBuildParameters parameters{1.2, 32, 64, 64, 32, true}; - auto index = svs::index::vamana::MutableVamanaIndex( - parameters, std::move(data), ids, Distance(), num_threads - ); - - auto sp = svs::index::vamana::VamanaSearchParameters{}; - sp.buffer_config_ = svs::index::vamana::SearchBufferConfig{50}; - - // Runs the filtered search; `found` = number of non-padding ids per query. - auto search_filtered = [&](auto& results, - const std::function& filter, - std::vector& found) { - index.search( - results.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - &filter - ); - found.assign(num_queries, 0); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - found[q] += results.index(q, j) != static_cast(-1); - } - } - }; - - // Brute force over live ids passing `filter`. - std::unordered_set deleted; - auto brute_force = [&](size_t q, auto&& filter) { - std::vector> all; - for (size_t i = 0; i < n; ++i) { - size_t e = ids[i]; - if (deleted.contains(e) || !filter(e)) { - continue; - } - float d = 0; - for (size_t j = 0; j < dim; ++j) { - float diff = queries.get_datum(q)[j] - raw[i][j]; - d += diff * diff; - } - all.emplace_back(d, e); - } - std::sort(all.begin(), all.end()); - std::vector out; - for (size_t i = 0; i < std::min(k, all.size()); ++i) { - out.push_back(all[i].second); - } - return out; - }; - - auto check = [&](auto&& filter, bool expect_full) { - auto results = svs::QueryResult(num_queries, k); - std::vector found; - search_filtered(results, filter, found); - CATCH_REQUIRE(found.size() == num_queries); - size_t hits = 0, total = 0; - for (size_t q = 0; q < num_queries; ++q) { - auto gt = brute_force(q, filter); - CATCH_REQUIRE(found[q] <= k); - if (expect_full) { - CATCH_REQUIRE(found[q] == k); - } - for (size_t j = 0; j < found[q]; ++j) { - size_t id = results.index(q, j); - CATCH_REQUIRE(filter(id)); - CATCH_REQUIRE(!deleted.contains(id)); - hits += std::count(gt.begin(), gt.end(), id); - } - for (size_t j = found[q]; j < k; ++j) { - CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); - CATCH_REQUIRE( - results.distance(q, j) == std::numeric_limits::infinity() - ); - } - total += gt.size(); - } - return double(hits) / double(total); - }; - - auto mod10 = [](size_t e) { return e % 10 == 0; }; - - CATCH_SECTION("recall against brute force") { - CATCH_REQUIRE(check(mod10, true) >= 0.95); - } - - CATCH_SECTION("deleted ids are never returned") { - std::vector to_delete; - for (size_t i = 0; i < n; i += 3) { - to_delete.push_back(ids[i]); - } - index.delete_entries(to_delete); - deleted.insert(to_delete.begin(), to_delete.end()); - CATCH_REQUIRE(check(mod10, true) >= 0.9); - CATCH_REQUIRE(check([](size_t) { return true; }, true) >= 0.9); - } - - CATCH_SECTION("fewer than k passing ids are padded") { - auto few = [](size_t e) { return e == 100 || e == 200 || e == 300; }; - check(few, false); - auto results = svs::QueryResult(num_queries, k); - std::vector found; - search_filtered(results, few, found); - for (size_t q = 0; q < num_queries; ++q) { - CATCH_REQUIRE(found[q] == 3); - } - auto none = [](size_t) { return false; }; - search_filtered(results, none, found); - for (size_t q = 0; q < num_queries; ++q) { - CATCH_REQUIRE(found[q] == 0); - } - } - - CATCH_SECTION("unfiltered search is unchanged by the optional argument") { - index.delete_entries(std::vector{ids[0], ids[1], ids[2]}); - auto plain = svs::QueryResult(num_queries, k); - auto with_null = svs::QueryResult(num_queries, k); - index.search(plain.view(), queries.cview(), sp); - index.search( - with_null.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - nullptr - ); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - CATCH_REQUIRE(plain.index(q, j) == with_null.index(q, j)); - CATCH_REQUIRE(plain.distance(q, j) == with_null.distance(q, j)); - } - } - } - - CATCH_SECTION("inner product pads with the worst score (-inf)") { - auto data_ip = svs::data::BlockedData(n, dim); - for (size_t i = 0; i < n; ++i) { - data_ip.set_datum(i, raw[i]); - } - svs::index::vamana::VamanaBuildParameters parameters_ip{0.95, 32, 64, 64, 32, true}; - auto index_ip = svs::index::vamana::MutableVamanaIndex( - parameters_ip, std::move(data_ip), ids, svs::distance::DistanceIP(), num_threads - ); - std::function few = [](size_t e) { return e == 100 || e == 200; }; - auto results = svs::QueryResult(num_queries, k); - index_ip.search( - results.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - &few - ); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 2; j < k; ++j) { - CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); - CATCH_REQUIRE( - results.distance(q, j) == -std::numeric_limits::infinity() - ); - } - } - } - - CATCH_SECTION("orchestrator entry point") { - auto dyn = svs::DynamicVamana::build( - parameters, - [&] { - auto d = svs::data::SimpleData(n, dim); - for (size_t i = 0; i < n; ++i) { - d.set_datum(i, raw[i]); - } - return d; - }(), - ids, - Distance(), - num_threads - ); - auto results = svs::QueryResult(num_queries, k); - CATCH_REQUIRE(dyn.filtered_search( - results.view(), queries.cview(), sp, std::function(mod10) - )); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - CATCH_REQUIRE(mod10(results.index(q, j))); - } - } - } -} From 201e4ca0859e41db8e73a261ac5c12792e18ee80 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 2 Nov 2026 18:05:34 -0800 Subject: [PATCH 18/47] Share the search body between filtered and regular search --- include/svs/index/vamana/dynamic_index.h | 152 ++++++++--------------- 1 file changed, 55 insertions(+), 97 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 86215e7cf..f9ef5d9d1 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -604,40 +604,24 @@ class MutableVamanaIndex { // This is an internal method, mostly used to help implement the batch iterator. ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + // With `Filtered`, an entry is only valid if `(*filter)(external_id)` and the closure + // carries `filter` along for the code that tops up and pads the results. + template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) - ) const { - return [&, prefetch_parameters]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { - // Perform the greedy search using the provided resources. - greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(), - prefetch_parameters, - cancel - ); - // Take a pass over the search buffer to remove any deleted elements that - // might remain. - buffer.cleanup(); - }; - } - - auto filtered_greedy_search_closure( - GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel, - const std::function& filter + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) const { - return [&, prefetch_parameters]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { + auto search = [&, prefetch_parameters, filter]( + const auto& query, auto& accessor, auto& distance, auto& buffer + ) { + auto builder = [&]() { + if constexpr (Filtered) { + return FilteringValidBuilder{status_, translator_, *filter}; + } else { + return internal_search_builder(); + } + }(); // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -647,7 +631,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - FilteringValidBuilder{status_, translator_, filter}, + builder, prefetch_parameters, cancel ); @@ -655,6 +639,11 @@ class MutableVamanaIndex { // might remain. buffer.cleanup(); }; + if constexpr (Filtered) { + return extensions::FilteredSearch{search, filter}; + } else { + return search; + } } // Single Search @@ -689,35 +678,42 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { + auto body = [&](std::bool_constant) { + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + // Legalize search buffer for this search. + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure( + prefetch_parameters, cancel, filter + ), + *this, + cancel + ); + }; if (filter != nullptr) { - filtered_search_thread(results, queries, sp, cancel, *filter, is); - return; + [&]() __attribute__((noinline)) { body(std::true_type{}); } + (); + } else { + body(std::false_type{}); } - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - // Legalize search buffer for this search. - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - greedy_search_closure(prefetch_parameters, cancel), - *this, - cancel - ); } ); @@ -731,44 +727,6 @@ class MutableVamanaIndex { translate_to_external(results.indices()); } - private: - // The part of `search` that handles a filter. Kept apart so that the unfiltered search - // above is compiled with the regular builder only. - template - SVS_NOINLINE void filtered_search_thread( - QueryResultView& results, - const Queries& queries, - const search_parameters_type& sp, - const lib::DefaultPredicate& cancel, - const std::function& filter, - const Range& is - ) { - size_t num_neighbors = results.n_neighbors(); - // Legalize search buffer for this search. - auto config = sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_; - auto buffer = search_buffer_type{ - config, distance::comparator(distance_), sp.search_buffer_visited_set_}; - - auto prefetch_parameters = - GreedySearchPrefetchParameters{sp.prefetch_lookahead_, sp.prefetch_step_}; - auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - auto search = filtered_greedy_search_closure(prefetch_parameters, cancel, filter); - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - extensions::FilteredSearch{search, &filter}, - *this, - cancel - ); - } - - public: /// /// @brief Return a unique instance of the distance function. /// From 7b71a60f8eb3a0ffb9ed24b4c758632ead6c7098 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 2 Nov 2026 19:12:41 -0800 Subject: [PATCH 19/47] Return to the ValidBuilder design before the compile-time changes --- include/svs/index/vamana/dynamic_index.h | 109 +++++------ tests/svs/index/vamana/dynamic_index.cpp | 222 +++++++++++++++++++++++ 2 files changed, 263 insertions(+), 68 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index f9ef5d9d1..f93f5c8dc 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -94,29 +94,13 @@ inline constexpr std::string_view name(SlotMetadata metadata) { class ValidBuilder { public: - ValidBuilder(const std::vector& status) - : status_{status} {} - - template - constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - bool invalid = getindex(status_, i) == SlotMetadata::Deleted; - // This neighbor should be skipped if the metadata corresponding to the given index - // marks this slot as deleted. - return PredicatedSearchNeighbor(i, distance, !invalid); - } - - private: - const std::vector& status_; -}; - -// Like `ValidBuilder`, but an entry is also invalid if `filter(external_id)` is false. -// Invalid entries are navigated through but never returned. Used by filtered searches. -class FilteringValidBuilder { - public: - FilteringValidBuilder( + // Optional `filter` (with `translator`): an entry is also invalid if + // `filter(external_id)` is false. Invalid entries are navigated through but never + // returned. + ValidBuilder( const std::vector& status, - const IDTranslator& translator, - const std::function& filter + const IDTranslator* translator = nullptr, + const std::function* filter = nullptr ) : status_{status} , translator_{translator} @@ -124,15 +108,19 @@ class FilteringValidBuilder { template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - bool invalid = getindex(status_, i) == SlotMetadata::Deleted || - !filter_(translator_.get_external(i)); + bool invalid = getindex(status_, i) == SlotMetadata::Deleted; + // This neighbor should be skipped if the metadata corresponding to the given index + // marks this slot as deleted. + if (!invalid && filter_ != nullptr) { + invalid = !(*filter_)(translator_->get_external(i)); + } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const std::vector& status_; - const IDTranslator& translator_; - const std::function& filter_; + const IDTranslator* translator_; + const std::function* filter_; }; template @@ -602,26 +590,19 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + ValidBuilder + internal_search_builder(const std::function* filter = nullptr) const { + return ValidBuilder{status_, &translator_, filter}; + } - // With `Filtered`, an entry is only valid if `(*filter)(external_id)` and the closure - // carries `filter` along for the code that tops up and pads the results. - template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - auto search = [&, prefetch_parameters, filter]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { - auto builder = [&]() { - if constexpr (Filtered) { - return FilteringValidBuilder{status_, translator_, *filter}; - } else { - return internal_search_builder(); - } - }(); + return [&, + prefetch_parameters, + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -631,7 +612,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - builder, + internal_search_builder(filter), prefetch_parameters, cancel ); @@ -639,11 +620,6 @@ class MutableVamanaIndex { // might remain. buffer.cleanup(); }; - if constexpr (Filtered) { - return extensions::FilteredSearch{search, filter}; - } else { - return search; - } } // Single Search @@ -678,22 +654,21 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - auto body = [&](std::bool_constant) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - // Legalize search buffer for this search. - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + // Legalize search buffer for this search. + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); + auto run = [&](const auto& search) { extensions::per_thread_batch_search( data_, buffer, @@ -701,18 +676,16 @@ class MutableVamanaIndex { queries, results, threads::UnitRange{is}, - greedy_search_closure( - prefetch_parameters, cancel, filter - ), + search, *this, cancel ); }; - if (filter != nullptr) { - [&]() __attribute__((noinline)) { body(std::true_type{}); } - (); + auto search = greedy_search_closure(prefetch_parameters, cancel, filter); + if (filter == nullptr) { + run(search); } else { - body(std::false_type{}); + run(extensions::FilteredSearch{search, filter}); } } ); diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 94daddcf1..6bedb8ea8 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -23,11 +23,14 @@ #include #include #include +#include #include #include +#include #include #include #include +#include // svs #include "svs/core/recall.h" @@ -464,3 +467,222 @@ CATCH_TEST_CASE("MutableVamana Index Memory Usage", "[graph_index][dynamic_index const size_t usage = index.get_memory_breakdown().total(); CATCH_REQUIRE(usage == expected_total_bytes); } + +CATCH_TEST_CASE( + "MutableVamana Index Filtered Search", "[graph_index][dynamic_index][filtered]" +) { + const size_t num_threads = 2; + const size_t n = 3000; + const size_t dim = 32; + const size_t num_queries = 50; + const size_t k = 10; + using Distance = svs::distance::DistanceL2; + + std::mt19937 rng(42); + std::normal_distribution normal; + auto data = svs::data::BlockedData(n, dim); + std::vector> raw(n, std::vector(dim)); + for (size_t i = 0; i < n; ++i) { + for (auto& x : raw[i]) { + x = normal(rng); + } + data.set_datum(i, raw[i]); + } + auto queries = svs::data::SimpleData(num_queries, dim); + for (size_t i = 0; i < num_queries; ++i) { + std::vector q(dim); + for (auto& x : q) { + x = normal(rng); + } + queries.set_datum(i, q); + } + + // External ids differ from internal ids. + std::vector ids(n); + std::iota(ids.begin(), ids.end(), 7); + svs::index::vamana::VamanaBuildParameters parameters{1.2, 32, 64, 64, 32, true}; + auto index = svs::index::vamana::MutableVamanaIndex( + parameters, std::move(data), ids, Distance(), num_threads + ); + + auto sp = svs::index::vamana::VamanaSearchParameters{}; + sp.buffer_config_ = svs::index::vamana::SearchBufferConfig{50}; + + // Runs the filtered search; `found` = number of non-padding ids per query. + auto search_filtered = [&](auto& results, + const std::function& filter, + std::vector& found) { + index.search( + results.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + &filter + ); + found.assign(num_queries, 0); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 0; j < k; ++j) { + found[q] += results.index(q, j) != static_cast(-1); + } + } + }; + + // Brute force over live ids passing `filter`. + std::unordered_set deleted; + auto brute_force = [&](size_t q, auto&& filter) { + std::vector> all; + for (size_t i = 0; i < n; ++i) { + size_t e = ids[i]; + if (deleted.contains(e) || !filter(e)) { + continue; + } + float d = 0; + for (size_t j = 0; j < dim; ++j) { + float diff = queries.get_datum(q)[j] - raw[i][j]; + d += diff * diff; + } + all.emplace_back(d, e); + } + std::sort(all.begin(), all.end()); + std::vector out; + for (size_t i = 0; i < std::min(k, all.size()); ++i) { + out.push_back(all[i].second); + } + return out; + }; + + auto check = [&](auto&& filter, bool expect_full) { + auto results = svs::QueryResult(num_queries, k); + std::vector found; + search_filtered(results, filter, found); + CATCH_REQUIRE(found.size() == num_queries); + size_t hits = 0, total = 0; + for (size_t q = 0; q < num_queries; ++q) { + auto gt = brute_force(q, filter); + CATCH_REQUIRE(found[q] <= k); + if (expect_full) { + CATCH_REQUIRE(found[q] == k); + } + for (size_t j = 0; j < found[q]; ++j) { + size_t id = results.index(q, j); + CATCH_REQUIRE(filter(id)); + CATCH_REQUIRE(!deleted.contains(id)); + hits += std::count(gt.begin(), gt.end(), id); + } + for (size_t j = found[q]; j < k; ++j) { + CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); + CATCH_REQUIRE( + results.distance(q, j) == std::numeric_limits::infinity() + ); + } + total += gt.size(); + } + return double(hits) / double(total); + }; + + auto mod10 = [](size_t e) { return e % 10 == 0; }; + + CATCH_SECTION("recall against brute force") { + CATCH_REQUIRE(check(mod10, true) >= 0.95); + } + + CATCH_SECTION("deleted ids are never returned") { + std::vector to_delete; + for (size_t i = 0; i < n; i += 3) { + to_delete.push_back(ids[i]); + } + index.delete_entries(to_delete); + deleted.insert(to_delete.begin(), to_delete.end()); + CATCH_REQUIRE(check(mod10, true) >= 0.9); + CATCH_REQUIRE(check([](size_t) { return true; }, true) >= 0.9); + } + + CATCH_SECTION("fewer than k passing ids are padded") { + auto few = [](size_t e) { return e == 100 || e == 200 || e == 300; }; + check(few, false); + auto results = svs::QueryResult(num_queries, k); + std::vector found; + search_filtered(results, few, found); + for (size_t q = 0; q < num_queries; ++q) { + CATCH_REQUIRE(found[q] == 3); + } + auto none = [](size_t) { return false; }; + search_filtered(results, none, found); + for (size_t q = 0; q < num_queries; ++q) { + CATCH_REQUIRE(found[q] == 0); + } + } + + CATCH_SECTION("unfiltered search is unchanged by the optional argument") { + index.delete_entries(std::vector{ids[0], ids[1], ids[2]}); + auto plain = svs::QueryResult(num_queries, k); + auto with_null = svs::QueryResult(num_queries, k); + index.search(plain.view(), queries.cview(), sp); + index.search( + with_null.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + nullptr + ); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 0; j < k; ++j) { + CATCH_REQUIRE(plain.index(q, j) == with_null.index(q, j)); + CATCH_REQUIRE(plain.distance(q, j) == with_null.distance(q, j)); + } + } + } + + CATCH_SECTION("inner product pads with the worst score (-inf)") { + auto data_ip = svs::data::BlockedData(n, dim); + for (size_t i = 0; i < n; ++i) { + data_ip.set_datum(i, raw[i]); + } + svs::index::vamana::VamanaBuildParameters parameters_ip{0.95, 32, 64, 64, 32, true}; + auto index_ip = svs::index::vamana::MutableVamanaIndex( + parameters_ip, std::move(data_ip), ids, svs::distance::DistanceIP(), num_threads + ); + std::function few = [](size_t e) { return e == 100 || e == 200; }; + auto results = svs::QueryResult(num_queries, k); + index_ip.search( + results.view(), + queries.cview(), + sp, + svs::lib::Returns(svs::lib::Const()), + &few + ); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 2; j < k; ++j) { + CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); + CATCH_REQUIRE( + results.distance(q, j) == -std::numeric_limits::infinity() + ); + } + } + } + + CATCH_SECTION("orchestrator entry point") { + auto dyn = svs::DynamicVamana::build( + parameters, + [&] { + auto d = svs::data::SimpleData(n, dim); + for (size_t i = 0; i < n; ++i) { + d.set_datum(i, raw[i]); + } + return d; + }(), + ids, + Distance(), + num_threads + ); + auto results = svs::QueryResult(num_queries, k); + CATCH_REQUIRE(dyn.filtered_search( + results.view(), queries.cview(), sp, std::function(mod10) + )); + for (size_t q = 0; q < num_queries; ++q) { + for (size_t j = 0; j < k; ++j) { + CATCH_REQUIRE(mod10(results.index(q, j))); + } + } + } +} From 71b6bf41c9fb3406af0ae479c616a8136e35039d Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 2 Nov 2026 19:20:53 -0800 Subject: [PATCH 20/47] Choose filtered or regular search at compile time --- include/svs/index/vamana/dynamic_index.h | 85 ++++++++++++++---------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index f93f5c8dc..fd840f237 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,7 +92,7 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on -class ValidBuilder { +template class ValidBuilder { public: // Optional `filter` (with `translator`): an entry is also invalid if // `filter(external_id)` is false. Invalid entries are navigated through but never @@ -111,8 +111,10 @@ class ValidBuilder { bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. - if (!invalid && filter_ != nullptr) { - invalid = !(*filter_)(translator_->get_external(i)); + if constexpr (Filtered) { + if (!invalid) { + invalid = !(*filter_)(translator_->get_external(i)); + } } return PredicatedSearchNeighbor(i, distance, !invalid); } @@ -590,11 +592,13 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder + template + ValidBuilder internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &translator_, filter}; + return ValidBuilder{status_, &translator_, filter}; } + template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), @@ -612,7 +616,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(filter), + internal_search_builder(filter), prefetch_parameters, cancel ); @@ -654,38 +658,49 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - // Legalize search buffer for this search. - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - - auto run = [&](const auto& search) { - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel + auto body = [&](std::bool_constant) { + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + // Legalize search buffer for this search. + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + auto run = [&](const auto& search) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); + }; + auto search = greedy_search_closure( + prefetch_parameters, cancel, filter ); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter}); + } else { + run(search); + } }; - auto search = greedy_search_closure(prefetch_parameters, cancel, filter); - if (filter == nullptr) { - run(search); + if (filter != nullptr) { + [&]() __attribute__((noinline)) { body(std::true_type{}); } + (); } else { - run(extensions::FilteredSearch{search, filter}); + body(std::false_type{}); } } ); From a78c6b011e76410aa7bf8a1fb9d517e775518d2e Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Tue, 3 Nov 2026 13:27:24 -0800 Subject: [PATCH 21/47] Filtered search on the concurrent index --- include/svs/concurrent/dynamic_index.h | 138 ++++++++++++++++++------- 1 file changed, 99 insertions(+), 39 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 3ef21e8b4..46d459a0f 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,10 +147,19 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -class ValidBuilder { +template class ValidBuilder { public: - ValidBuilder(const lib::SegmentedVector& status) - : status_{status} {} + // Optional `filter` (with `index`): an entry is also invalid if + // `filter(external_id)` is false. Invalid entries are navigated through but never + // returned. + ValidBuilder( + const lib::SegmentedVector& status, + const Index* index = nullptr, + const std::function* filter = nullptr + ) + : status_{status} + , index_{index} + , filter_{filter} {} template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { @@ -163,11 +172,21 @@ class ValidBuilder { .load(std::memory_order_acquire) != SlotMetadata::Valid; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. + if constexpr (Filtered) { + if (!invalid) { + // Locks the translation per lookup so writers keep running. An id erased + // concurrently has no external id and fails the filter. + size_t external = index_->translate_internal_id(i); + invalid = external == no_external_id || !(*filter_)(external); + } + } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const lib::SegmentedVector& status_; + const Index* index_; + const std::function* filter_; }; template @@ -804,6 +823,9 @@ class MutableVamanaIndex { [&](const auto is, uint64_t /*tid*/) { for (auto i : is) { for (size_t j = 0, jmax = getsize<1>(ids); j < jmax; ++j) { + if (ids.at(i, j) == static_cast(-1)) { + continue; // Padding from a filtered search. + } auto internal = lib::narrow_cast(ids.at(i, j)); ids.at(i, j) = unsafe_translate_internal_id(internal); } @@ -874,7 +896,11 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + template + ValidBuilder + internal_search_builder(const std::function* filter = nullptr) const { + return ValidBuilder{status_, this, filter}; + } /// @brief RAII reader lock guarding data_/graph_ against compact()'s shrink /// (which frees segments). Used by BatchIterator::next() to protect the @@ -894,13 +920,15 @@ class MutableVamanaIndex { return std::shared_lock(*translator_mutex_); } + template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) const { - return [&, prefetch_parameters]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { + return [&, + prefetch_parameters, + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { const Idx seed = entry_point(); if (seed == NO_ENTRY) { buffer.clear(); @@ -915,7 +943,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{std::span{&seed, 1}}, - internal_search_builder(), + internal_search_builder(filter), prefetch_parameters, cancel ); @@ -941,7 +969,7 @@ class MutableVamanaIndex { scratch.buffer, scratch.scratch, query, - greedy_search_closure(scratch.prefetch_parameters, cancel), + greedy_search_closure(scratch.prefetch_parameters, cancel), *this ); } @@ -951,7 +979,8 @@ class MutableVamanaIndex { QueryResultView results, const Queries& queries, const search_parameters_type& sp, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) { // Internal IDs must retain their meaning until translation has completed. std::shared_lock compact_lock{*compact_mutex_}; @@ -960,33 +989,52 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - // Legalize search buffer for this search. - if (buffer.target_capacity() < num_neighbors) { - buffer.change_maxsize(num_neighbors); + auto body = [&](std::bool_constant) { + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + // Legalize search buffer for this search. + if (buffer.target_capacity() < num_neighbors) { + buffer.change_maxsize(num_neighbors); + } + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + auto run = [&](const auto& search) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); + }; + auto search = greedy_search_closure( + prefetch_parameters, cancel, filter + ); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter} + ); + } else { + run(search); + } + }; + if (filter != nullptr) { + [&]() __attribute__((noinline)) { body(std::true_type{}); } + (); + } else { + body(std::false_type{}); } - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - greedy_search_closure(prefetch_parameters, cancel), - *this, - cancel - ); } ); } @@ -2087,7 +2135,11 @@ class MutableVamanaIndex { /// Called from the ``extensions::single_search`` override below. /// template - void supplement_search_buffer(SearchBuffer& search_buffer, const Query& query) const { + void supplement_search_buffer( + SearchBuffer& search_buffer, + const Query& query, + const std::function* filter = nullptr + ) const { if (search_buffer.valid() >= search_buffer.target_window() || search_buffer.valid() >= size()) { return; @@ -2100,6 +2152,10 @@ class MutableVamanaIndex { auto builder = internal_search_builder(); for (auto pair : translator_) { auto internal_id = pair.second; + // `pair.first` is the external id, read under the lock held above. + if (filter != nullptr && !(*filter)(pair.first)) { + continue; + } // Skip slots pending deferred translator cleanup, reserved by an in-flight // add_points, or otherwise not live. if (std::atomic_ref(const_cast(status_[internal_id] @@ -2190,7 +2246,11 @@ SVS_FORCE_INLINE void svs_invoke( if (cancel()) { return; } - index.supplement_search_buffer(search_buffer, query); + const std::function* filter = nullptr; + if constexpr (requires { search.filter; }) { + filter = search.filter; // Filtered search: keep filtered-out ids out of the top-up. + } + index.supplement_search_buffer(search_buffer, query, filter); } ///// Deduction Guides. From 697dac9fe625fbad311131cc5dec6f90123a7281 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Tue, 3 Nov 2026 16:10:29 -0800 Subject: [PATCH 22/47] Skip the filter for entries the buffer would drop --- include/svs/concurrent/dynamic_index.h | 21 ++++++++++++++++++++- include/svs/index/vamana/dynamic_index.h | 21 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 46d459a0f..588a130f9 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -161,8 +161,21 @@ template class ValidBuilder { , index_{index} , filter_{filter} {} + /// Filtered only: skip the filter for entries `buffer` would drop anyway. + template void skip_if_dropped_by(const Buffer& buffer) { + skip_buffer_ = &buffer; + skip_ = [](const void* b, float d) { + return static_cast(b)->can_skip(d); + }; + } + template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + if constexpr (Filtered) { + if (skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); + } + } // A neighbor is returnable only if its slot is Valid. Deleted slots // must be skipped; Pending slots are reserved by an in-flight add and // their vectors/edges are not yet fully published. Empty slots should @@ -187,6 +200,8 @@ template class ValidBuilder { const lib::SegmentedVector& status_; const Index* index_; const std::function* filter_; + const void* skip_buffer_ = nullptr; + bool (*skip_)(const void*, float) = nullptr; }; template @@ -934,6 +949,10 @@ class MutableVamanaIndex { buffer.clear(); return; } + auto builder = internal_search_builder(filter); + if constexpr (Filtered) { + builder.skip_if_dropped_by(buffer); + } // Perform the greedy search using the provided resources. concurrent::greedy_search( graph_, @@ -943,7 +962,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{std::span{&seed, 1}}, - internal_search_builder(filter), + builder, prefetch_parameters, cancel ); diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index fd840f237..b0015c2ac 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -106,8 +106,21 @@ template class ValidBuilder { , translator_{translator} , filter_{filter} {} + /// Filtered only: skip the filter for entries `buffer` would drop anyway. + template void skip_if_dropped_by(const Buffer& buffer) { + skip_buffer_ = &buffer; + skip_ = [](const void* b, float d) { + return static_cast(b)->can_skip(d); + }; + } + template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + if constexpr (Filtered) { + if (skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); + } + } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. @@ -123,6 +136,8 @@ template class ValidBuilder { const std::vector& status_; const IDTranslator* translator_; const std::function* filter_; + const void* skip_buffer_ = nullptr; + bool (*skip_)(const void*, float) = nullptr; }; template @@ -607,6 +622,10 @@ class MutableVamanaIndex { return [&, prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + auto builder = internal_search_builder(filter); + if constexpr (Filtered) { + builder.skip_if_dropped_by(buffer); + } // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -616,7 +635,7 @@ class MutableVamanaIndex { distance, buffer, vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(filter), + builder, prefetch_parameters, cancel ); From 9f5f98ee7c2cb05889c97b0268e3dc715db42d80 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Tue, 3 Nov 2026 18:08:17 -0800 Subject: [PATCH 23/47] Read external ids without a lock in filtered search --- include/svs/concurrent/dynamic_index.h | 56 +++++++++++++++++++++----- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 588a130f9..80951a157 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,18 +147,18 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -template class ValidBuilder { +template class ValidBuilder { public: // Optional `filter` (with `index`): an entry is also invalid if // `filter(external_id)` is false. Invalid entries are navigated through but never // returned. ValidBuilder( const lib::SegmentedVector& status, - const Index* index = nullptr, + const lib::SegmentedVector* external_ids = nullptr, const std::function* filter = nullptr ) : status_{status} - , index_{index} + , external_ids_{external_ids} , filter_{filter} {} /// Filtered only: skip the filter for entries `buffer` would drop anyway. @@ -187,9 +187,11 @@ template class ValidBuilder { // marks this slot as deleted. if constexpr (Filtered) { if (!invalid) { - // Locks the translation per lookup so writers keep running. An id erased - // concurrently has no external id and fails the filter. - size_t external = index_->translate_internal_id(i); + // Lock-free: the id was stored before the slot's Valid release above. + size_t external = + std::atomic_ref(const_cast(getindex(*external_ids_, i)) + ) + .load(std::memory_order_acquire); invalid = external == no_external_id || !(*filter_)(external); } } @@ -198,7 +200,7 @@ template class ValidBuilder { private: const lib::SegmentedVector& status_; - const Index* index_; + const lib::SegmentedVector* external_ids_; const std::function* filter_; const void* skip_buffer_ = nullptr; bool (*skip_)(const void*, float) = nullptr; @@ -254,6 +256,9 @@ class MutableVamanaIndex { // Grow-stable per-slot metadata: search reads status_[i] lock-free via ValidBuilder // while a concurrent add_points grows it. See svs/lib/segmented_vector.h. lib::SegmentedVector status_; + // External id of each slot (or no_external_id); mirrors translator_ for lock-free + // reads. + lib::SegmentedVector external_ids_; // a lower bound such that the append path // (reuse_empty == false) scans for Empty slots starting here. std::unique_ptr> first_empty_{ @@ -332,6 +337,7 @@ class MutableVamanaIndex { , data_{std::move(data)} , entry_point_{data_.size() == 0 ? NO_ENTRY : entry_point} , status_(data_.size(), SlotMetadata::Valid) + , external_ids_(data_.size(), no_external_id) , first_empty_{std::make_unique>(data_.size())} , first_reusable_{std::make_unique>(data_.size())} , translator_() @@ -349,6 +355,9 @@ class MutableVamanaIndex { throw ANNEXCEPTION("Graph node count does not match external IDs size"); } translator_.insert(external_ids, threads::UnitRange(0, external_ids.size())); + for (auto pair : translator_) { + external_ids_[pair.second] = pair.first; + } } public: @@ -491,6 +500,7 @@ class MutableVamanaIndex { , data_{std::move(data)} , entry_point_{data_.size() == 0 ? NO_ENTRY : lib::narrow(config.entry_point)} , status_{data_.size(), SlotMetadata::Empty} // filled later in the body + , external_ids_{data_.size(), no_external_id} , first_empty_{std::make_unique>(data_.size())} , first_reusable_{std::make_unique>(data_.size())} , translator_{std::move(translator)} @@ -556,6 +566,7 @@ class MutableVamanaIndex { pair.second ); } + external_ids_[pair.second] = pair.first; } num_valid_->store(num_valid); @@ -866,7 +877,10 @@ class MutableVamanaIndex { return ReplaceExternalIdResult::OldIdMissing; } try { + auto slot = translator_.get_internal(old_id); translator_.remap_external_id(old_id, new_id); + std::atomic_ref(external_ids_[slot]) + .store(new_id, std::memory_order_relaxed); } catch (const ANNException&) { return ReplaceExternalIdResult::NewIdExists; } return ReplaceExternalIdResult::Ok; } @@ -912,9 +926,9 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. template - ValidBuilder + ValidBuilder internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, this, filter}; + return ValidBuilder{status_, &external_ids_, filter}; } /// @brief RAII reader lock guarding data_/graph_ against compact()'s shrink @@ -1220,6 +1234,7 @@ class MutableVamanaIndex { data_.resize(new_size); graph_.unsafe_resize(new_size); status_.resize(new_size, SlotMetadata::Empty); + external_ids_.resize(new_size, no_external_id); for (size_t i = current_size; i < new_size; ++i) { std::atomic_ref(status_[i]) .store(SlotMetadata::Pending, std::memory_order_release); @@ -1240,6 +1255,12 @@ class MutableVamanaIndex { // maps to a live slot. try { std::lock_guard lock{*translator_mutex_}; + std::vector stale{}; + for (auto e : external_ids) { + if (translator_.has_external(e)) { + stale.push_back(translator_.get_internal(e)); + } + } translator_ .replace_stale_and_insert(external_ids, slots, [this](auto internal) { return std::atomic_ref( @@ -1247,6 +1268,17 @@ class MutableVamanaIndex { ) .load(std::memory_order_acquire) == SlotMetadata::Deleted; }); + // Slots whose mapping was just erased must not keep a now-live id. + for (auto s : stale) { + std::atomic_ref(external_ids_[s]) + .store(no_external_id, std::memory_order_relaxed); + } + // Stored before the slots turn Valid (release), which readers pair with. + size_t j = 0; + for (auto e : external_ids) { + std::atomic_ref(external_ids_[slots[j++]]) + .store(e, std::memory_order_relaxed); + } } catch (...) { // Release the reserved slots back to Empty. for (auto s : slots) { @@ -1591,11 +1623,13 @@ class MutableVamanaIndex { auto status = getindex(status_, old_id); status_[new_id] = status; + external_ids_[new_id] = getindex(external_ids_, old_id); if (status == SlotMetadata::Valid) { translator_.remap_internal_id(old_id, new_id); } } status_.resize(max_index); + external_ids_.resize(max_index); // Remap the surviving anchor, or publish the genuinely empty state. auto ep = old_to_new_id_map.find(entry_point()); @@ -1730,6 +1764,10 @@ class MutableVamanaIndex { } } if (!deleted_internal_ids.empty()) { + for (auto i : deleted_internal_ids) { + std::atomic_ref(external_ids_[i]) + .store(no_external_id, std::memory_order_relaxed); + } translator_.delete_internal(deleted_internal_ids, false); } // The anchor's external mapping is removed above like any deletion, From c066008cc189e68f7b3f6fa72af967849a45458a Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 4 Nov 2026 09:32:23 -0800 Subject: [PATCH 24/47] Use the translator shared lock once per search in filtered search --- include/svs/concurrent/dynamic_index.h | 71 ++++++++------------------ 1 file changed, 21 insertions(+), 50 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 80951a157..64d3cf5ba 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,18 +147,18 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -template class ValidBuilder { +template class ValidBuilder { public: // Optional `filter` (with `index`): an entry is also invalid if // `filter(external_id)` is false. Invalid entries are navigated through but never // returned. ValidBuilder( const lib::SegmentedVector& status, - const lib::SegmentedVector* external_ids = nullptr, + const Index* index = nullptr, const std::function* filter = nullptr ) : status_{status} - , external_ids_{external_ids} + , index_{index} , filter_{filter} {} /// Filtered only: skip the filter for entries `buffer` would drop anyway. @@ -187,11 +187,9 @@ template class ValidBuilder { // marks this slot as deleted. if constexpr (Filtered) { if (!invalid) { - // Lock-free: the id was stored before the slot's Valid release above. - size_t external = - std::atomic_ref(const_cast(getindex(*external_ids_, i)) - ) - .load(std::memory_order_acquire); + // The caller holds `lock_for_translation()`. An id erased before the lock + // was taken has no external id and fails the filter. + size_t external = index_->unsafe_translate_internal_id(i); invalid = external == no_external_id || !(*filter_)(external); } } @@ -200,7 +198,7 @@ template class ValidBuilder { private: const lib::SegmentedVector& status_; - const lib::SegmentedVector* external_ids_; + const Index* index_; const std::function* filter_; const void* skip_buffer_ = nullptr; bool (*skip_)(const void*, float) = nullptr; @@ -256,9 +254,6 @@ class MutableVamanaIndex { // Grow-stable per-slot metadata: search reads status_[i] lock-free via ValidBuilder // while a concurrent add_points grows it. See svs/lib/segmented_vector.h. lib::SegmentedVector status_; - // External id of each slot (or no_external_id); mirrors translator_ for lock-free - // reads. - lib::SegmentedVector external_ids_; // a lower bound such that the append path // (reuse_empty == false) scans for Empty slots starting here. std::unique_ptr> first_empty_{ @@ -337,7 +332,6 @@ class MutableVamanaIndex { , data_{std::move(data)} , entry_point_{data_.size() == 0 ? NO_ENTRY : entry_point} , status_(data_.size(), SlotMetadata::Valid) - , external_ids_(data_.size(), no_external_id) , first_empty_{std::make_unique>(data_.size())} , first_reusable_{std::make_unique>(data_.size())} , translator_() @@ -355,9 +349,6 @@ class MutableVamanaIndex { throw ANNEXCEPTION("Graph node count does not match external IDs size"); } translator_.insert(external_ids, threads::UnitRange(0, external_ids.size())); - for (auto pair : translator_) { - external_ids_[pair.second] = pair.first; - } } public: @@ -500,7 +491,6 @@ class MutableVamanaIndex { , data_{std::move(data)} , entry_point_{data_.size() == 0 ? NO_ENTRY : lib::narrow(config.entry_point)} , status_{data_.size(), SlotMetadata::Empty} // filled later in the body - , external_ids_{data_.size(), no_external_id} , first_empty_{std::make_unique>(data_.size())} , first_reusable_{std::make_unique>(data_.size())} , translator_{std::move(translator)} @@ -566,7 +556,6 @@ class MutableVamanaIndex { pair.second ); } - external_ids_[pair.second] = pair.first; } num_valid_->store(num_valid); @@ -877,10 +866,7 @@ class MutableVamanaIndex { return ReplaceExternalIdResult::OldIdMissing; } try { - auto slot = translator_.get_internal(old_id); translator_.remap_external_id(old_id, new_id); - std::atomic_ref(external_ids_[slot]) - .store(new_id, std::memory_order_relaxed); } catch (const ANNException&) { return ReplaceExternalIdResult::NewIdExists; } return ReplaceExternalIdResult::Ok; } @@ -926,9 +912,9 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. template - ValidBuilder + ValidBuilder internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &external_ids_, filter}; + return ValidBuilder{status_, this, filter}; } /// @brief RAII reader lock guarding data_/graph_ against compact()'s shrink @@ -1007,6 +993,8 @@ class MutableVamanaIndex { ); } + // `filter` runs while this thread holds the translator lock shared: it must not call + // back into this index (no writers, no self-locking readers such as `has_id`). template void search( QueryResultView results, @@ -1056,6 +1044,8 @@ class MutableVamanaIndex { prefetch_parameters, cancel, filter ); if constexpr (Filtered) { + // Held once for this thread's queries, not per lookup. + auto translation_lock = lock_for_translation(); run(extensions::FilteredSearch{search, filter} ); } else { @@ -1234,7 +1224,6 @@ class MutableVamanaIndex { data_.resize(new_size); graph_.unsafe_resize(new_size); status_.resize(new_size, SlotMetadata::Empty); - external_ids_.resize(new_size, no_external_id); for (size_t i = current_size; i < new_size; ++i) { std::atomic_ref(status_[i]) .store(SlotMetadata::Pending, std::memory_order_release); @@ -1255,12 +1244,6 @@ class MutableVamanaIndex { // maps to a live slot. try { std::lock_guard lock{*translator_mutex_}; - std::vector stale{}; - for (auto e : external_ids) { - if (translator_.has_external(e)) { - stale.push_back(translator_.get_internal(e)); - } - } translator_ .replace_stale_and_insert(external_ids, slots, [this](auto internal) { return std::atomic_ref( @@ -1268,17 +1251,6 @@ class MutableVamanaIndex { ) .load(std::memory_order_acquire) == SlotMetadata::Deleted; }); - // Slots whose mapping was just erased must not keep a now-live id. - for (auto s : stale) { - std::atomic_ref(external_ids_[s]) - .store(no_external_id, std::memory_order_relaxed); - } - // Stored before the slots turn Valid (release), which readers pair with. - size_t j = 0; - for (auto e : external_ids) { - std::atomic_ref(external_ids_[slots[j++]]) - .store(e, std::memory_order_relaxed); - } } catch (...) { // Release the reserved slots back to Empty. for (auto s : slots) { @@ -1623,13 +1595,11 @@ class MutableVamanaIndex { auto status = getindex(status_, old_id); status_[new_id] = status; - external_ids_[new_id] = getindex(external_ids_, old_id); if (status == SlotMetadata::Valid) { translator_.remap_internal_id(old_id, new_id); } } status_.resize(max_index); - external_ids_.resize(max_index); // Remap the surviving anchor, or publish the genuinely empty state. auto ep = old_to_new_id_map.find(entry_point()); @@ -1764,10 +1734,6 @@ class MutableVamanaIndex { } } if (!deleted_internal_ids.empty()) { - for (auto i : deleted_internal_ids) { - std::atomic_ref(external_ids_[i]) - .store(no_external_id, std::memory_order_relaxed); - } translator_.delete_internal(deleted_internal_ids, false); } // The anchor's external mapping is removed above like any deletion, @@ -2189,7 +2155,8 @@ class MutableVamanaIndex { /// ``translator_mutex_`` shared (preserving the global compact -> translator order) /// and skipping slots that are no longer ``Valid``. /// - /// Called from the ``extensions::single_search`` override below. + /// Called from the ``extensions::single_search`` override below. With a ``filter`` the + /// caller must hold ``lock_for_translation()`` (``search()`` does). /// template void supplement_search_buffer( @@ -2204,8 +2171,12 @@ class MutableVamanaIndex { search_buffer.sort(); // translator_mutex_ shared: guards both the traversal of ``translator_`` and the - // internal IDs it yields against erasure by consolidate()/compact(). - std::shared_lock translator_lock{*translator_mutex_}; + // internal IDs it yields against erasure by consolidate()/compact(). A filtered + // search already holds it (see ``search()``); locking again would be recursive. + std::shared_lock translator_lock{*translator_mutex_, std::defer_lock}; + if (filter == nullptr) { + translator_lock.lock(); + } auto builder = internal_search_builder(); for (auto pair : translator_) { auto internal_id = pair.second; From bc56c0e05c861d002f43b17862d53c2f48ce52b6 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Wed, 4 Nov 2026 13:37:24 -0800 Subject: [PATCH 25/47] Read internal-to-external ids from a grow-stable vector (concurrent index, PR #447 idea) --- include/svs/concurrent/dynamic_index.h | 46 +++++++---- include/svs/concurrent/translation.h | 102 ++++++++++++++++--------- 2 files changed, 98 insertions(+), 50 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 64d3cf5ba..d42e2ed39 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -187,8 +187,8 @@ template class ValidBuilder { // marks this slot as deleted. if constexpr (Filtered) { if (!invalid) { - // The caller holds `lock_for_translation()`. An id erased before the lock - // was taken has no external id and fails the filter. + // Lock-free read of the internal->external table. A slot whose id was + // erased has no external id and fails the filter. size_t external = index_->unsafe_translate_internal_id(i); invalid = external == no_external_id || !(*filter_)(external); } @@ -763,7 +763,8 @@ class MutableVamanaIndex { } /// @copydoc translate_internal_id - /// Requires the caller to hold `lock_for_translation()`. + /// Requires the caller to hold `lock_for_translation()`, except in filtered search + /// where the internal->external table is read lock-free. size_t unsafe_translate_internal_id(Idx i) const { // Not `i` itself: an internal ID is no label, and a caller would report it as one. return translator_.get_external_or(i, no_external_id); @@ -1044,8 +1045,6 @@ class MutableVamanaIndex { prefetch_parameters, cancel, filter ); if constexpr (Filtered) { - // Held once for this thread's queries, not per lookup. - auto translation_lock = lock_for_translation(); run(extensions::FilteredSearch{search, filter} ); } else { @@ -1600,6 +1599,7 @@ class MutableVamanaIndex { } } status_.resize(max_index); + translator_.shrink(max_index); // Remap the surviving anchor, or publish the genuinely empty state. auto ep = old_to_new_id_map.find(entry_point()); @@ -2156,7 +2156,8 @@ class MutableVamanaIndex { /// and skipping slots that are no longer ``Valid``. /// /// Called from the ``extensions::single_search`` override below. With a ``filter`` the - /// caller must hold ``lock_for_translation()`` (``search()`` does). + /// scan walks the slots and reads their ids lock-free, so the filter runs without + /// ``translator_mutex_`` held. /// template void supplement_search_buffer( @@ -2170,20 +2171,33 @@ class MutableVamanaIndex { } search_buffer.sort(); - // translator_mutex_ shared: guards both the traversal of ``translator_`` and the - // internal IDs it yields against erasure by consolidate()/compact(). A filtered - // search already holds it (see ``search()``); locking again would be recursive. - std::shared_lock translator_lock{*translator_mutex_, std::defer_lock}; - if (filter == nullptr) { - translator_lock.lock(); + if (filter != nullptr) { + // compact_mutex_ shared (held by the caller) keeps the slots in place. + auto builder = internal_search_builder(); + for (size_t i = 0, n = status_.size(); i < n; ++i) { + if (std::atomic_ref(const_cast(status_[i])) + .load(std::memory_order_acquire) != SlotMetadata::Valid) { + continue; + } + auto external = unsafe_translate_internal_id(i); + if (external == no_external_id || !(*filter)(external)) { + continue; + } + auto dist = extensions::get_distance_ext(data_, distance_, i, query); + search_buffer.insert(builder(static_cast(i), dist)); + if (search_buffer.valid() >= search_buffer.target_window()) { + break; + } + } + return; } + + // translator_mutex_ shared: guards both the traversal of ``translator_`` and the + // internal IDs it yields against erasure by consolidate()/compact(). + std::shared_lock translator_lock{*translator_mutex_}; auto builder = internal_search_builder(); for (auto pair : translator_) { auto internal_id = pair.second; - // `pair.first` is the external id, read under the lock held above. - if (filter != nullptr && !(*filter)(pair.first)) { - continue; - } // Skip slots pending deferred translator cleanup, reserved by an in-flight // add_points, or otherwise not live. if (std::atomic_ref(const_cast(status_[internal_id] diff --git a/include/svs/concurrent/translation.h b/include/svs/concurrent/translation.h index 5306689e4..b65eb5591 100644 --- a/include/svs/concurrent/translation.h +++ b/include/svs/concurrent/translation.h @@ -27,6 +27,7 @@ #include "svs/lib/narrow.h" #include "svs/lib/readwrite.h" #include "svs/lib/saveload.h" +#include "svs/lib/segmented_vector.h" #include "svs/lib/threads.h" #include "svs/third-party/fmt.h" @@ -35,8 +36,10 @@ #include "tsl/robin_map.h" // stl +#include #include #include +#include #include namespace svs::index::vamana::concurrent { @@ -50,6 +53,10 @@ class IDTranslator { tsl::robin_map::const_iterator; using value_type = typename const_iterator::value_type; + /// Marks an unmapped slot in the internal-to-external table. + static constexpr external_id_type invalid_external_id = + std::numeric_limits::max(); + // Construct the identity transformation of size `n`. struct Identity { Identity() = delete; @@ -67,17 +74,13 @@ class IDTranslator { /// /// @brief Return the number of translations. /// - size_t size() const { - if constexpr (checkbounds_v) { - const size_t e2i = external_to_internal_.size(); - const size_t i2e = internal_to_external_.size(); + size_t size() const { return external_to_internal_.size(); } - if (e2i != i2e) { - throw ANNEXCEPTION("Size mismatch! E2I is {} while I2E is {}!", e2i, i2e); - } - } - return external_to_internal_.size(); - } + /// + /// @brief Drop the internal-to-external slots ``[n, ...)``. No internal ID ``>= n`` may + /// be mapped, and no reader may be active (see ``lib::SegmentedVector::shrink_to``). + /// + void shrink(size_t n) { internal_to_external_.shrink_to(n); } /// /// @brief Insert the two ranges. @@ -141,7 +144,7 @@ class IDTranslator { template void insert_translation(Ext external_id, Int internal_id) { external_to_internal_[external_id] = lib::narrow(internal_id); - internal_to_external_[internal_id] = lib::narrow(external_id); + store_external(internal_id, lib::narrow(external_id)); } /// @@ -195,7 +198,7 @@ class IDTranslator { for (; e != ext_end; ++e, ++i) { auto found = external_to_internal_.find(*e); if (found != external_to_internal_.end()) { - internal_to_external_.erase(found->second); + store_external(found->second, invalid_external_id); } insert_translation(*e, *i); } @@ -216,7 +219,7 @@ class IDTranslator { /// @param e The internal ID to check. /// bool has_internal(internal_id_type e) const { - return internal_to_external_.contains(e); + return load_external(e) != invalid_external_id; } /// @@ -234,17 +237,18 @@ class IDTranslator { /// @param i The internal ID to translate to an external ID. /// external_id_type get_external(internal_id_type i) const { - return internal_to_external_.at(i); + auto e = load_external(i); + if (e == invalid_external_id) { + throw std::out_of_range("IDTranslator: internal ID not found"); + } + return e; } /// @brief Return the external ID, or a default if not found. external_id_type get_external_or(internal_id_type i, external_id_type default_val) const { - auto it = internal_to_external_.find(i); - if (it == internal_to_external_.end()) { - return default_val; - } - return it->second; + auto e = load_external(i); + return e == invalid_external_id ? default_val : e; } /// @brief Return the internal ID, or a default if not found. @@ -278,13 +282,12 @@ class IDTranslator { assert(has_internal(from)); assert(!has_internal(to)); - auto itr = internal_to_external_.find(from); - auto external = itr->second; + auto external = load_external(from); // Updating the internal-to-external ID is easy. external_to_internal_[external] = to; - internal_to_external_.erase(itr); - internal_to_external_.insert({to, external}); + store_external(from, invalid_external_id); + store_external(to, external); } /// @@ -309,7 +312,7 @@ class IDTranslator { auto internal = itr->second; // Updating the external-to-internal ID is easy. - internal_to_external_[internal] = to; + store_external(internal, to); external_to_internal_.erase(itr); external_to_internal_.insert({to, internal}); } @@ -335,7 +338,7 @@ class IDTranslator { for (const auto i : internal_ids) { auto e = get_external(i); - internal_to_external_.erase(i); + store_external(i, invalid_external_id); external_to_internal_.erase(e); } } @@ -361,7 +364,7 @@ class IDTranslator { for (const auto e : external_ids) { auto i = get_internal(e); - internal_to_external_.erase(i); + store_external(i, invalid_external_id); external_to_internal_.erase(e); } } @@ -374,7 +377,12 @@ class IDTranslator { /// template void check_external_free(const Begin& begin, const End& end) const { - check(begin, end, external_to_internal_, "Index already contains external"); + check( + begin, + end, + [this](auto e) { return has_external(e); }, + "Index already contains external" + ); } /// @@ -388,7 +396,7 @@ class IDTranslator { check( begin, end, - external_to_internal_, + [this](auto e) { return has_external(e); }, "Index does not contain external", std::logical_not() ); @@ -402,7 +410,12 @@ class IDTranslator { /// template void check_internal_free(const Begin& begin, const End& end) const { - check(begin, end, internal_to_external_, "Index already contains internal"); + check( + begin, + end, + [this](auto i) { return has_internal(i); }, + "Index already contains internal" + ); } /// @@ -416,7 +429,7 @@ class IDTranslator { check( begin, end, - internal_to_external_, + [this](auto i) { return has_internal(i); }, "Index does not contain internal", std::logical_not() ); @@ -500,23 +513,44 @@ class IDTranslator { } private: - template + template void check( const Begin& begin, const End& end, - const Map& map, + const Contains& contains, const char* message, Modifier modify = lib::identity() ) const { for (auto i = begin; i != end; ++i) { - if (modify(map.contains(*i))) { + if (modify(contains(*i))) { throw ANNEXCEPTION("{} ID {}!", message, *i); } } } tsl::robin_map external_to_internal_{}; - tsl::robin_map internal_to_external_{}; + // Indexed by internal ID. Grow-stable, so `load_external` is safe without a lock while + // a writer (holding the index's exclusive translator lock) grows it or stores entries. + lib::SegmentedVector internal_to_external_{}; + + external_id_type load_external(internal_id_type i) const { + if (i >= internal_to_external_.size()) { + return invalid_external_id; + } + return std::atomic_ref( + const_cast(internal_to_external_[i]) + ) + .load(std::memory_order_acquire); + } + + // Writers only; grows the table first when `i` is beyond it. + void store_external(size_t i, external_id_type e) { + if (i >= internal_to_external_.size()) { + internal_to_external_.resize(i + 1, invalid_external_id); + } + std::atomic_ref(internal_to_external_[i]) + .store(e, std::memory_order_release); + } }; } // namespace svs::index::vamana::concurrent From bf6f5c4d95fe6164cdaf942392173803159e0f24 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 11:34:36 -0800 Subject: [PATCH 26/47] Remove the filtered-search test changes --- tests/svs/index/vamana/dynamic_index.cpp | 222 ----------------------- 1 file changed, 222 deletions(-) diff --git a/tests/svs/index/vamana/dynamic_index.cpp b/tests/svs/index/vamana/dynamic_index.cpp index 6bedb8ea8..94daddcf1 100644 --- a/tests/svs/index/vamana/dynamic_index.cpp +++ b/tests/svs/index/vamana/dynamic_index.cpp @@ -23,14 +23,11 @@ #include #include #include -#include #include #include -#include #include #include #include -#include // svs #include "svs/core/recall.h" @@ -467,222 +464,3 @@ CATCH_TEST_CASE("MutableVamana Index Memory Usage", "[graph_index][dynamic_index const size_t usage = index.get_memory_breakdown().total(); CATCH_REQUIRE(usage == expected_total_bytes); } - -CATCH_TEST_CASE( - "MutableVamana Index Filtered Search", "[graph_index][dynamic_index][filtered]" -) { - const size_t num_threads = 2; - const size_t n = 3000; - const size_t dim = 32; - const size_t num_queries = 50; - const size_t k = 10; - using Distance = svs::distance::DistanceL2; - - std::mt19937 rng(42); - std::normal_distribution normal; - auto data = svs::data::BlockedData(n, dim); - std::vector> raw(n, std::vector(dim)); - for (size_t i = 0; i < n; ++i) { - for (auto& x : raw[i]) { - x = normal(rng); - } - data.set_datum(i, raw[i]); - } - auto queries = svs::data::SimpleData(num_queries, dim); - for (size_t i = 0; i < num_queries; ++i) { - std::vector q(dim); - for (auto& x : q) { - x = normal(rng); - } - queries.set_datum(i, q); - } - - // External ids differ from internal ids. - std::vector ids(n); - std::iota(ids.begin(), ids.end(), 7); - svs::index::vamana::VamanaBuildParameters parameters{1.2, 32, 64, 64, 32, true}; - auto index = svs::index::vamana::MutableVamanaIndex( - parameters, std::move(data), ids, Distance(), num_threads - ); - - auto sp = svs::index::vamana::VamanaSearchParameters{}; - sp.buffer_config_ = svs::index::vamana::SearchBufferConfig{50}; - - // Runs the filtered search; `found` = number of non-padding ids per query. - auto search_filtered = [&](auto& results, - const std::function& filter, - std::vector& found) { - index.search( - results.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - &filter - ); - found.assign(num_queries, 0); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - found[q] += results.index(q, j) != static_cast(-1); - } - } - }; - - // Brute force over live ids passing `filter`. - std::unordered_set deleted; - auto brute_force = [&](size_t q, auto&& filter) { - std::vector> all; - for (size_t i = 0; i < n; ++i) { - size_t e = ids[i]; - if (deleted.contains(e) || !filter(e)) { - continue; - } - float d = 0; - for (size_t j = 0; j < dim; ++j) { - float diff = queries.get_datum(q)[j] - raw[i][j]; - d += diff * diff; - } - all.emplace_back(d, e); - } - std::sort(all.begin(), all.end()); - std::vector out; - for (size_t i = 0; i < std::min(k, all.size()); ++i) { - out.push_back(all[i].second); - } - return out; - }; - - auto check = [&](auto&& filter, bool expect_full) { - auto results = svs::QueryResult(num_queries, k); - std::vector found; - search_filtered(results, filter, found); - CATCH_REQUIRE(found.size() == num_queries); - size_t hits = 0, total = 0; - for (size_t q = 0; q < num_queries; ++q) { - auto gt = brute_force(q, filter); - CATCH_REQUIRE(found[q] <= k); - if (expect_full) { - CATCH_REQUIRE(found[q] == k); - } - for (size_t j = 0; j < found[q]; ++j) { - size_t id = results.index(q, j); - CATCH_REQUIRE(filter(id)); - CATCH_REQUIRE(!deleted.contains(id)); - hits += std::count(gt.begin(), gt.end(), id); - } - for (size_t j = found[q]; j < k; ++j) { - CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); - CATCH_REQUIRE( - results.distance(q, j) == std::numeric_limits::infinity() - ); - } - total += gt.size(); - } - return double(hits) / double(total); - }; - - auto mod10 = [](size_t e) { return e % 10 == 0; }; - - CATCH_SECTION("recall against brute force") { - CATCH_REQUIRE(check(mod10, true) >= 0.95); - } - - CATCH_SECTION("deleted ids are never returned") { - std::vector to_delete; - for (size_t i = 0; i < n; i += 3) { - to_delete.push_back(ids[i]); - } - index.delete_entries(to_delete); - deleted.insert(to_delete.begin(), to_delete.end()); - CATCH_REQUIRE(check(mod10, true) >= 0.9); - CATCH_REQUIRE(check([](size_t) { return true; }, true) >= 0.9); - } - - CATCH_SECTION("fewer than k passing ids are padded") { - auto few = [](size_t e) { return e == 100 || e == 200 || e == 300; }; - check(few, false); - auto results = svs::QueryResult(num_queries, k); - std::vector found; - search_filtered(results, few, found); - for (size_t q = 0; q < num_queries; ++q) { - CATCH_REQUIRE(found[q] == 3); - } - auto none = [](size_t) { return false; }; - search_filtered(results, none, found); - for (size_t q = 0; q < num_queries; ++q) { - CATCH_REQUIRE(found[q] == 0); - } - } - - CATCH_SECTION("unfiltered search is unchanged by the optional argument") { - index.delete_entries(std::vector{ids[0], ids[1], ids[2]}); - auto plain = svs::QueryResult(num_queries, k); - auto with_null = svs::QueryResult(num_queries, k); - index.search(plain.view(), queries.cview(), sp); - index.search( - with_null.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - nullptr - ); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - CATCH_REQUIRE(plain.index(q, j) == with_null.index(q, j)); - CATCH_REQUIRE(plain.distance(q, j) == with_null.distance(q, j)); - } - } - } - - CATCH_SECTION("inner product pads with the worst score (-inf)") { - auto data_ip = svs::data::BlockedData(n, dim); - for (size_t i = 0; i < n; ++i) { - data_ip.set_datum(i, raw[i]); - } - svs::index::vamana::VamanaBuildParameters parameters_ip{0.95, 32, 64, 64, 32, true}; - auto index_ip = svs::index::vamana::MutableVamanaIndex( - parameters_ip, std::move(data_ip), ids, svs::distance::DistanceIP(), num_threads - ); - std::function few = [](size_t e) { return e == 100 || e == 200; }; - auto results = svs::QueryResult(num_queries, k); - index_ip.search( - results.view(), - queries.cview(), - sp, - svs::lib::Returns(svs::lib::Const()), - &few - ); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 2; j < k; ++j) { - CATCH_REQUIRE(results.index(q, j) == static_cast(-1)); - CATCH_REQUIRE( - results.distance(q, j) == -std::numeric_limits::infinity() - ); - } - } - } - - CATCH_SECTION("orchestrator entry point") { - auto dyn = svs::DynamicVamana::build( - parameters, - [&] { - auto d = svs::data::SimpleData(n, dim); - for (size_t i = 0; i < n; ++i) { - d.set_datum(i, raw[i]); - } - return d; - }(), - ids, - Distance(), - num_threads - ); - auto results = svs::QueryResult(num_queries, k); - CATCH_REQUIRE(dyn.filtered_search( - results.view(), queries.cview(), sp, std::function(mod10) - )); - for (size_t q = 0; q < num_queries; ++q) { - for (size_t j = 0; j < k; ++j) { - CATCH_REQUIRE(mod10(results.index(q, j))); - } - } - } -} From 66719ffb51321cb9f7753faa1a2973fcddce3ff2 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 12:24:19 -0800 Subject: [PATCH 27/47] Remove dead code and trim comments in filtered search --- include/svs/concurrent/dynamic_index.h | 35 +++++++--------- include/svs/concurrent/translation.h | 10 +---- include/svs/index/vamana/dynamic_index.h | 10 ++--- include/svs/index/vamana/extensions.h | 3 +- include/svs/orchestrators/dynamic_vamana.h | 49 ++++++++-------------- 5 files changed, 39 insertions(+), 68 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index d42e2ed39..c411f30d9 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,21 +147,20 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -template class ValidBuilder { +// Filtered is a template parameter so regular search compiles without any filter check. +template class ValidBuilder { public: - // Optional `filter` (with `index`): an entry is also invalid if - // `filter(external_id)` is false. Invalid entries are navigated through but never - // returned. + // Filtered: entries failing filter(external_id) are navigated but not returned. ValidBuilder( const lib::SegmentedVector& status, - const Index* index = nullptr, + const IDTranslator* translator = nullptr, const std::function* filter = nullptr ) : status_{status} - , index_{index} + , translator_{translator} , filter_{filter} {} - /// Filtered only: skip the filter for entries `buffer` would drop anyway. + // Once the window is full, only candidates closer than its worst entry are filtered. template void skip_if_dropped_by(const Buffer& buffer) { skip_buffer_ = &buffer; skip_ = [](const void* b, float d) { @@ -189,7 +188,7 @@ template class ValidBuilder { if (!invalid) { // Lock-free read of the internal->external table. A slot whose id was // erased has no external id and fails the filter. - size_t external = index_->unsafe_translate_internal_id(i); + size_t external = translator_->get_external_or(i, no_external_id); invalid = external == no_external_id || !(*filter_)(external); } } @@ -198,7 +197,7 @@ template class ValidBuilder { private: const lib::SegmentedVector& status_; - const Index* index_; + const IDTranslator* translator_; const std::function* filter_; const void* skip_buffer_ = nullptr; bool (*skip_)(const void*, float) = nullptr; @@ -763,8 +762,7 @@ class MutableVamanaIndex { } /// @copydoc translate_internal_id - /// Requires the caller to hold `lock_for_translation()`, except in filtered search - /// where the internal->external table is read lock-free. + /// Requires the caller to hold `lock_for_translation()`. size_t unsafe_translate_internal_id(Idx i) const { // Not `i` itself: an internal ID is no label, and a caller would report it as one. return translator_.get_external_or(i, no_external_id); @@ -913,9 +911,9 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. template - ValidBuilder + ValidBuilder internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, this, filter}; + return ValidBuilder{status_, &translator_, filter}; } /// @brief RAII reader lock guarding data_/graph_ against compact()'s shrink @@ -994,8 +992,8 @@ class MutableVamanaIndex { ); } - // `filter` runs while this thread holds the translator lock shared: it must not call - // back into this index (no writers, no self-locking readers such as `has_id`). + // `filter` runs on every search thread without the translator lock; it must be + // thread-safe. template void search( QueryResultView results, @@ -1599,7 +1597,6 @@ class MutableVamanaIndex { } } status_.resize(max_index); - translator_.shrink(max_index); // Remap the surviving anchor, or publish the genuinely empty state. auto ep = old_to_new_id_map.find(entry_point()); @@ -2155,9 +2152,7 @@ class MutableVamanaIndex { /// ``translator_mutex_`` shared (preserving the global compact -> translator order) /// and skipping slots that are no longer ``Valid``. /// - /// Called from the ``extensions::single_search`` override below. With a ``filter`` the - /// scan walks the slots and reads their ids lock-free, so the filter runs without - /// ``translator_mutex_`` held. + /// Called from the ``extensions::single_search`` override below. /// template void supplement_search_buffer( @@ -2290,7 +2285,7 @@ SVS_FORCE_INLINE void svs_invoke( } const std::function* filter = nullptr; if constexpr (requires { search.filter; }) { - filter = search.filter; // Filtered search: keep filtered-out ids out of the top-up. + filter = search.filter; } index.supplement_search_buffer(search_buffer, query, filter); } diff --git a/include/svs/concurrent/translation.h b/include/svs/concurrent/translation.h index b65eb5591..652c2c7b7 100644 --- a/include/svs/concurrent/translation.h +++ b/include/svs/concurrent/translation.h @@ -76,12 +76,6 @@ class IDTranslator { /// size_t size() const { return external_to_internal_.size(); } - /// - /// @brief Drop the internal-to-external slots ``[n, ...)``. No internal ID ``>= n`` may - /// be mapped, and no reader may be active (see ``lib::SegmentedVector::shrink_to``). - /// - void shrink(size_t n) { internal_to_external_.shrink_to(n); } - /// /// @brief Insert the two ranges. /// @@ -529,8 +523,8 @@ class IDTranslator { } tsl::robin_map external_to_internal_{}; - // Indexed by internal ID. Grow-stable, so `load_external` is safe without a lock while - // a writer (holding the index's exclusive translator lock) grows it or stores entries. + // TODO: moves to #447 (same vector-based internal-to-external table); not in this PR. + // Grow-stable: `load_external` is safe without a lock. lib::SegmentedVector internal_to_external_{}; external_id_type load_external(internal_id_type i) const { diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index b0015c2ac..0ffa8e09d 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,11 +92,10 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on +// Filtered is a template parameter so regular search compiles without any filter check. template class ValidBuilder { public: - // Optional `filter` (with `translator`): an entry is also invalid if - // `filter(external_id)` is false. Invalid entries are navigated through but never - // returned. + // Filtered: entries failing filter(external_id) are navigated but not returned. ValidBuilder( const std::vector& status, const IDTranslator* translator = nullptr, @@ -106,7 +105,7 @@ template class ValidBuilder { , translator_{translator} , filter_{filter} {} - /// Filtered only: skip the filter for entries `buffer` would drop anyway. + // Once the window is full, only candidates closer than its worst entry are filtered. template void skip_if_dropped_by(const Buffer& buffer) { skip_buffer_ = &buffer; skip_ = [](const void* b, float d) { @@ -670,9 +669,6 @@ class MutableVamanaIndex { const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) { - // Optional `filter`: only entries whose external id passes are returned. It is - // called from all search threads. Rows with fewer passing entries than requested - // are padded with id `size_t(-1)` and the worst score. threads::parallel_for( threadpool_, threads::StaticPartition{queries.size()}, diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 9ec594486..a745a7401 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -501,8 +501,7 @@ SVS_FORCE_INLINE void svs_invoke( if constexpr (Index::needs_id_translation) { const std::function* filter = nullptr; if constexpr (requires { search.filter; }) { - filter = - search.filter; // Filtered search: keep filtered-out ids out of the top-up. + filter = search.filter; } check_and_supplement_search_buffer(index, search_buffer, query, filter); } diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 1fb30f041..4aa705c08 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -124,36 +124,23 @@ class DynamicVamanaImpl : public VamanaImpl& filter ) override { - // Filtered search is available for every data type. Indexes whose `search` has no - // filter argument (e.g. the concurrent index) report "unsupported". - constexpr bool supported = requires( - Impl & idx, - svs::QueryResultView r, - data::ConstSimpleDataView q, - const index::vamana::VamanaSearchParameters& sp, - const std::function* filter - ) { idx.search(r, q, sp, lib::Returns(lib::Const()), filter); }; - if constexpr (supported) { - lib::match( - QueryTypes{}, - queries.type(), - [&](lib::Type SVS_UNUSED(type)) { - impl().search( - results, - data::ConstSimpleDataView(queries), - search_parameters, - lib::Returns(lib::Const()), - &filter - ); - }, - [&](svs::DataType data_type) { - throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); - } - ); - return true; - } else { - return false; - } + lib::match( + QueryTypes{}, + queries.type(), + [&](lib::Type SVS_UNUSED(type)) { + impl().search( + results, + data::ConstSimpleDataView(queries), + search_parameters, + lib::Returns(lib::Const()), + &filter + ); + }, + [&](svs::DataType data_type) { + throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); + } + ); + return true; } }; @@ -657,7 +644,7 @@ class DynamicVamana : public manager::IndexManager { bool is_consolidated() const { return impl_->is_consolidated(); } /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. - /// Returns false if the data type does not support it. + /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. template bool filtered_search( QueryResultView results, From 1a3aacb0171d0266b15677c5bbd431f1e71607c2 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 14:23:28 -0800 Subject: [PATCH 28/47] Clarify the ValidBuilder comment --- include/svs/concurrent/dynamic_index.h | 3 ++- include/svs/index/vamana/dynamic_index.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index c411f30d9..9c27d48f5 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -150,7 +150,8 @@ inline constexpr size_t no_external_id = // Filtered is a template parameter so regular search compiles without any filter check. template class ValidBuilder { public: - // Filtered: entries failing filter(external_id) are navigated but not returned. + // Filtered: a candidate failing filter(external_id) still gets its distance computed + // and its neighbors visited, but is marked invalid and never returned. ValidBuilder( const lib::SegmentedVector& status, const IDTranslator* translator = nullptr, diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 0ffa8e09d..cac37a8ea 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -95,7 +95,8 @@ inline constexpr std::string_view name(SlotMetadata metadata) { // Filtered is a template parameter so regular search compiles without any filter check. template class ValidBuilder { public: - // Filtered: entries failing filter(external_id) are navigated but not returned. + // Filtered: a candidate failing filter(external_id) still gets its distance computed + // and its neighbors visited, but is marked invalid and never returned. ValidBuilder( const std::vector& status, const IDTranslator* translator = nullptr, From cf574cb3225cbf53dbe26f7034f977ec6f555758 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 14:27:50 -0800 Subject: [PATCH 29/47] Put the ValidBuilder comment on the class --- include/svs/concurrent/dynamic_index.h | 5 ++--- include/svs/index/vamana/dynamic_index.h | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 9c27d48f5..db21b66e0 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,11 +147,10 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -// Filtered is a template parameter so regular search compiles without any filter check. +// Marks deleted (and, if Filtered, filtered-out) slots invalid: searched through, never +// returned; a template so regular search has no filter check. template class ValidBuilder { public: - // Filtered: a candidate failing filter(external_id) still gets its distance computed - // and its neighbors visited, but is marked invalid and never returned. ValidBuilder( const lib::SegmentedVector& status, const IDTranslator* translator = nullptr, diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index cac37a8ea..dfeec4959 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,11 +92,10 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on -// Filtered is a template parameter so regular search compiles without any filter check. +// Marks deleted (and, if Filtered, filtered-out) slots invalid: searched through, never +// returned; a template so regular search has no filter check. template class ValidBuilder { public: - // Filtered: a candidate failing filter(external_id) still gets its distance computed - // and its neighbors visited, but is marked invalid and never returned. ValidBuilder( const std::vector& status, const IDTranslator* translator = nullptr, From 8931142f469515754ccda7f9f8bce45a82030773 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 14:29:10 -0800 Subject: [PATCH 30/47] Shorten the ValidBuilder comment --- include/svs/concurrent/dynamic_index.h | 3 +-- include/svs/index/vamana/dynamic_index.h | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index db21b66e0..d9d97ae36 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,8 +147,7 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -// Marks deleted (and, if Filtered, filtered-out) slots invalid: searched through, never -// returned; a template so regular search has no filter check. +// Template so the compiler builds a separate regular-search copy with no filter check. template class ValidBuilder { public: ValidBuilder( diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index dfeec4959..6c4c84a66 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,8 +92,7 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on -// Marks deleted (and, if Filtered, filtered-out) slots invalid: searched through, never -// returned; a template so regular search has no filter check. +// Template so the compiler builds a separate regular-search copy with no filter check. template class ValidBuilder { public: ValidBuilder( From 73d7128e8f973ae1d6a64b5cfb2b5943c3a83e42 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 14:47:37 -0800 Subject: [PATCH 31/47] Minimal filtered search --- include/svs/concurrent/dynamic_index.h | 155 ++++++++------------- include/svs/index/vamana/dynamic_index.h | 140 +++++++------------ include/svs/orchestrators/dynamic_vamana.h | 19 +-- 3 files changed, 120 insertions(+), 194 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index d9d97ae36..b90dee01f 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,33 +147,13 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -// Template so the compiler builds a separate regular-search copy with no filter check. -template class ValidBuilder { +class ValidBuilder { public: - ValidBuilder( - const lib::SegmentedVector& status, - const IDTranslator* translator = nullptr, - const std::function* filter = nullptr - ) - : status_{status} - , translator_{translator} - , filter_{filter} {} - - // Once the window is full, only candidates closer than its worst entry are filtered. - template void skip_if_dropped_by(const Buffer& buffer) { - skip_buffer_ = &buffer; - skip_ = [](const void* b, float d) { - return static_cast(b)->can_skip(d); - }; - } + ValidBuilder(const lib::SegmentedVector& status) + : status_{status} {} template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if constexpr (Filtered) { - if (skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } - } // A neighbor is returnable only if its slot is Valid. Deleted slots // must be skipped; Pending slots are reserved by an in-flight add and // their vectors/edges are not yet fully published. Empty slots should @@ -183,23 +163,11 @@ template class ValidBuilder { .load(std::memory_order_acquire) != SlotMetadata::Valid; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. - if constexpr (Filtered) { - if (!invalid) { - // Lock-free read of the internal->external table. A slot whose id was - // erased has no external id and fails the filter. - size_t external = translator_->get_external_or(i, no_external_id); - invalid = external == no_external_id || !(*filter_)(external); - } - } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const lib::SegmentedVector& status_; - const IDTranslator* translator_; - const std::function* filter_; - const void* skip_buffer_ = nullptr; - bool (*skip_)(const void*, float) = nullptr; }; template @@ -836,9 +804,6 @@ class MutableVamanaIndex { [&](const auto is, uint64_t /*tid*/) { for (auto i : is) { for (size_t j = 0, jmax = getsize<1>(ids); j < jmax; ++j) { - if (ids.at(i, j) == static_cast(-1)) { - continue; // Padding from a filtered search. - } auto internal = lib::narrow_cast(ids.at(i, j)); ids.at(i, j) = unsafe_translate_internal_id(internal); } @@ -909,11 +874,7 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - template - ValidBuilder - internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &translator_, filter}; - } + ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } /// @brief RAII reader lock guarding data_/graph_ against compact()'s shrink /// (which frees segments). Used by BatchIterator::next() to protect the @@ -933,24 +894,36 @@ class MutableVamanaIndex { return std::shared_lock(*translator_mutex_); } + // Filtered: ids failing `filter` are traversed but never returned. template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - return [&, - prefetch_parameters, - filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + auto search = [&, prefetch_parameters, filter]( + const auto& query, auto& accessor, auto& distance, auto& buffer + ) { const Idx seed = entry_point(); if (seed == NO_ENTRY) { buffer.clear(); return; } - auto builder = internal_search_builder(filter); - if constexpr (Filtered) { - builder.skip_if_dropped_by(buffer); - } + auto valid = internal_search_builder(); + auto builder = [&](Idx i, float dist) { + auto neighbor = valid(i, dist); + if constexpr (Filtered) { + // The filter is not called for candidates a full buffer would drop. + if (neighbor.valid() && !buffer.can_skip(dist)) { + // Lock-free read; an erased slot has no external id. + size_t external = unsafe_translate_internal_id(i); + if (external == no_external_id || !(*filter)(external)) { + return PredicatedSearchNeighbor(i, dist, false); + } + } + } + return neighbor; + }; // Perform the greedy search using the provided resources. concurrent::greedy_search( graph_, @@ -968,6 +941,11 @@ class MutableVamanaIndex { // might remain. buffer.cleanup(); }; + if constexpr (Filtered) { + return extensions::FilteredSearch{search, filter}; + } else { + return search; + } } // The scratch buffer contains internal IDs. Consuming them after this call requires @@ -986,14 +964,14 @@ class MutableVamanaIndex { scratch.buffer, scratch.scratch, query, - greedy_search_closure(scratch.prefetch_parameters, cancel), + greedy_search_closure(scratch.prefetch_parameters, cancel), *this ); } // `filter` runs on every search thread without the translator lock; it must be // thread-safe. - template + template void search( QueryResultView results, const Queries& queries, @@ -1008,52 +986,35 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - auto body = [&](std::bool_constant) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - // Legalize search buffer for this search. - if (buffer.target_capacity() < num_neighbors) { - buffer.change_maxsize(num_neighbors); - } - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - auto run = [&](const auto& search) { - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; - auto search = greedy_search_closure( - prefetch_parameters, cancel, filter - ); - if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter} - ); - } else { - run(search); - } - }; - if (filter != nullptr) { - [&]() __attribute__((noinline)) { body(std::true_type{}); } - (); - } else { - body(std::false_type{}); + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + // Legalize search buffer for this search. + if (buffer.target_capacity() < num_neighbors) { + buffer.change_maxsize(num_neighbors); } + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure( + prefetch_parameters, cancel, filter + ), + *this, + cancel + ); } ); } diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 6c4c84a66..d42475487 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,50 +92,21 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on -// Template so the compiler builds a separate regular-search copy with no filter check. -template class ValidBuilder { +class ValidBuilder { public: - ValidBuilder( - const std::vector& status, - const IDTranslator* translator = nullptr, - const std::function* filter = nullptr - ) - : status_{status} - , translator_{translator} - , filter_{filter} {} - - // Once the window is full, only candidates closer than its worst entry are filtered. - template void skip_if_dropped_by(const Buffer& buffer) { - skip_buffer_ = &buffer; - skip_ = [](const void* b, float d) { - return static_cast(b)->can_skip(d); - }; - } + ValidBuilder(const std::vector& status) + : status_{status} {} template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if constexpr (Filtered) { - if (skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } - } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. - if constexpr (Filtered) { - if (!invalid) { - invalid = !(*filter_)(translator_->get_external(i)); - } - } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const std::vector& status_; - const IDTranslator* translator_; - const std::function* filter_; - const void* skip_buffer_ = nullptr; - bool (*skip_)(const void*, float) = nullptr; }; template @@ -605,25 +576,30 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - template - ValidBuilder - internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &translator_, filter}; - } + ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + // Filtered: ids failing `filter` are traversed but never returned. template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - return [&, - prefetch_parameters, - filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { - auto builder = internal_search_builder(filter); - if constexpr (Filtered) { - builder.skip_if_dropped_by(buffer); - } + auto search = [&, prefetch_parameters, filter]( + const auto& query, auto& accessor, auto& distance, auto& buffer + ) { + auto valid = internal_search_builder(); + auto builder = [&](Idx i, float dist) { + auto neighbor = valid(i, dist); + if constexpr (Filtered) { + // The filter is not called for candidates a full buffer would drop. + if (neighbor.valid() && !buffer.can_skip(dist) && + !(*filter)(translate_internal_id(i))) { + return PredicatedSearchNeighbor(i, dist, false); + } + } + return neighbor; + }; // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -641,6 +617,11 @@ class MutableVamanaIndex { // might remain. buffer.cleanup(); }; + if constexpr (Filtered) { + return extensions::FilteredSearch{search, filter}; + } else { + return search; + } } // Single Search @@ -660,7 +641,7 @@ class MutableVamanaIndex { ); } - template + template void search( QueryResultView results, const Queries& queries, @@ -672,50 +653,31 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - auto body = [&](std::bool_constant) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - // Legalize search buffer for this search. - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - auto run = [&](const auto& search) { - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; - auto search = greedy_search_closure( - prefetch_parameters, cancel, filter - ); - if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter}); - } else { - run(search); - } - }; - if (filter != nullptr) { - [&]() __attribute__((noinline)) { body(std::true_type{}); } - (); - } else { - body(std::false_type{}); - } + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + // Legalize search buffer for this search. + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); + + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure(prefetch_parameters, cancel, filter), + *this, + cancel + ); } ); diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 4aa705c08..f086dbe88 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -57,8 +57,8 @@ class DynamicVamanaInterface : public VamanaInterface { // Check if the index has been consolidated - no `Deleted` entries remain. virtual bool is_consolidated() const = 0; - // Filtered search; false if the data type does not support it. Keep last (ABI). - virtual bool filtered_search( + // Filtered search: only ids with filter(id) == true are returned. Keep last (ABI). + virtual void filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, @@ -118,7 +118,7 @@ class DynamicVamanaImpl : public VamanaImpl results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, @@ -128,7 +128,7 @@ class DynamicVamanaImpl : public VamanaImpl(lib::Type SVS_UNUSED(type)) { - impl().search( + impl().template search( results, data::ConstSimpleDataView(queries), search_parameters, @@ -137,10 +137,13 @@ class DynamicVamanaImpl : public VamanaImpl { /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. template - bool filtered_search( + void filtered_search( QueryResultView results, data::ConstSimpleDataView queries, const VamanaSearchParameters& search_parameters, const std::function& filter ) { - return impl_->filtered_search( + impl_->filtered_search( results, AnonymousArray<2>(queries), search_parameters, filter ); } From b5046f95c692af1d2aba116ac6fb72255684d53d Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 15:32:00 -0800 Subject: [PATCH 32/47] Revert "Minimal filtered search" This reverts commit 73d7128e8f973ae1d6a64b5cfb2b5943c3a83e42. --- include/svs/concurrent/dynamic_index.h | 155 +++++++++++++-------- include/svs/index/vamana/dynamic_index.h | 140 ++++++++++++------- include/svs/orchestrators/dynamic_vamana.h | 19 ++- 3 files changed, 194 insertions(+), 120 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index b90dee01f..d9d97ae36 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -147,13 +147,33 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -class ValidBuilder { +// Template so the compiler builds a separate regular-search copy with no filter check. +template class ValidBuilder { public: - ValidBuilder(const lib::SegmentedVector& status) - : status_{status} {} + ValidBuilder( + const lib::SegmentedVector& status, + const IDTranslator* translator = nullptr, + const std::function* filter = nullptr + ) + : status_{status} + , translator_{translator} + , filter_{filter} {} + + // Once the window is full, only candidates closer than its worst entry are filtered. + template void skip_if_dropped_by(const Buffer& buffer) { + skip_buffer_ = &buffer; + skip_ = [](const void* b, float d) { + return static_cast(b)->can_skip(d); + }; + } template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + if constexpr (Filtered) { + if (skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); + } + } // A neighbor is returnable only if its slot is Valid. Deleted slots // must be skipped; Pending slots are reserved by an in-flight add and // their vectors/edges are not yet fully published. Empty slots should @@ -163,11 +183,23 @@ class ValidBuilder { .load(std::memory_order_acquire) != SlotMetadata::Valid; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. + if constexpr (Filtered) { + if (!invalid) { + // Lock-free read of the internal->external table. A slot whose id was + // erased has no external id and fails the filter. + size_t external = translator_->get_external_or(i, no_external_id); + invalid = external == no_external_id || !(*filter_)(external); + } + } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const lib::SegmentedVector& status_; + const IDTranslator* translator_; + const std::function* filter_; + const void* skip_buffer_ = nullptr; + bool (*skip_)(const void*, float) = nullptr; }; template @@ -804,6 +836,9 @@ class MutableVamanaIndex { [&](const auto is, uint64_t /*tid*/) { for (auto i : is) { for (size_t j = 0, jmax = getsize<1>(ids); j < jmax; ++j) { + if (ids.at(i, j) == static_cast(-1)) { + continue; // Padding from a filtered search. + } auto internal = lib::narrow_cast(ids.at(i, j)); ids.at(i, j) = unsafe_translate_internal_id(internal); } @@ -874,7 +909,11 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + template + ValidBuilder + internal_search_builder(const std::function* filter = nullptr) const { + return ValidBuilder{status_, &translator_, filter}; + } /// @brief RAII reader lock guarding data_/graph_ against compact()'s shrink /// (which frees segments). Used by BatchIterator::next() to protect the @@ -894,36 +933,24 @@ class MutableVamanaIndex { return std::shared_lock(*translator_mutex_); } - // Filtered: ids failing `filter` are traversed but never returned. template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - auto search = [&, prefetch_parameters, filter]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { + return [&, + prefetch_parameters, + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { const Idx seed = entry_point(); if (seed == NO_ENTRY) { buffer.clear(); return; } - auto valid = internal_search_builder(); - auto builder = [&](Idx i, float dist) { - auto neighbor = valid(i, dist); - if constexpr (Filtered) { - // The filter is not called for candidates a full buffer would drop. - if (neighbor.valid() && !buffer.can_skip(dist)) { - // Lock-free read; an erased slot has no external id. - size_t external = unsafe_translate_internal_id(i); - if (external == no_external_id || !(*filter)(external)) { - return PredicatedSearchNeighbor(i, dist, false); - } - } - } - return neighbor; - }; + auto builder = internal_search_builder(filter); + if constexpr (Filtered) { + builder.skip_if_dropped_by(buffer); + } // Perform the greedy search using the provided resources. concurrent::greedy_search( graph_, @@ -941,11 +968,6 @@ class MutableVamanaIndex { // might remain. buffer.cleanup(); }; - if constexpr (Filtered) { - return extensions::FilteredSearch{search, filter}; - } else { - return search; - } } // The scratch buffer contains internal IDs. Consuming them after this call requires @@ -964,14 +986,14 @@ class MutableVamanaIndex { scratch.buffer, scratch.scratch, query, - greedy_search_closure(scratch.prefetch_parameters, cancel), + greedy_search_closure(scratch.prefetch_parameters, cancel), *this ); } // `filter` runs on every search thread without the translator lock; it must be // thread-safe. - template + template void search( QueryResultView results, const Queries& queries, @@ -986,35 +1008,52 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - // Legalize search buffer for this search. - if (buffer.target_capacity() < num_neighbors) { - buffer.change_maxsize(num_neighbors); - } - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - greedy_search_closure( + auto body = [&](std::bool_constant) { + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + // Legalize search buffer for this search. + if (buffer.target_capacity() < num_neighbors) { + buffer.change_maxsize(num_neighbors); + } + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + auto run = [&](const auto& search) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); + }; + auto search = greedy_search_closure( prefetch_parameters, cancel, filter - ), - *this, - cancel - ); + ); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter} + ); + } else { + run(search); + } + }; + if (filter != nullptr) { + [&]() __attribute__((noinline)) { body(std::true_type{}); } + (); + } else { + body(std::false_type{}); + } } ); } diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index d42475487..6c4c84a66 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,21 +92,50 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on -class ValidBuilder { +// Template so the compiler builds a separate regular-search copy with no filter check. +template class ValidBuilder { public: - ValidBuilder(const std::vector& status) - : status_{status} {} + ValidBuilder( + const std::vector& status, + const IDTranslator* translator = nullptr, + const std::function* filter = nullptr + ) + : status_{status} + , translator_{translator} + , filter_{filter} {} + + // Once the window is full, only candidates closer than its worst entry are filtered. + template void skip_if_dropped_by(const Buffer& buffer) { + skip_buffer_ = &buffer; + skip_ = [](const void* b, float d) { + return static_cast(b)->can_skip(d); + }; + } template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + if constexpr (Filtered) { + if (skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); + } + } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; // This neighbor should be skipped if the metadata corresponding to the given index // marks this slot as deleted. + if constexpr (Filtered) { + if (!invalid) { + invalid = !(*filter_)(translator_->get_external(i)); + } + } return PredicatedSearchNeighbor(i, distance, !invalid); } private: const std::vector& status_; + const IDTranslator* translator_; + const std::function* filter_; + const void* skip_buffer_ = nullptr; + bool (*skip_)(const void*, float) = nullptr; }; template @@ -576,30 +605,25 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + template + ValidBuilder + internal_search_builder(const std::function* filter = nullptr) const { + return ValidBuilder{status_, &translator_, filter}; + } - // Filtered: ids failing `filter` are traversed but never returned. template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), const std::function* filter = nullptr ) const { - auto search = [&, prefetch_parameters, filter]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { - auto valid = internal_search_builder(); - auto builder = [&](Idx i, float dist) { - auto neighbor = valid(i, dist); - if constexpr (Filtered) { - // The filter is not called for candidates a full buffer would drop. - if (neighbor.valid() && !buffer.can_skip(dist) && - !(*filter)(translate_internal_id(i))) { - return PredicatedSearchNeighbor(i, dist, false); - } - } - return neighbor; - }; + return [&, + prefetch_parameters, + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + auto builder = internal_search_builder(filter); + if constexpr (Filtered) { + builder.skip_if_dropped_by(buffer); + } // Perform the greedy search using the provided resources. greedy_search( graph_, @@ -617,11 +641,6 @@ class MutableVamanaIndex { // might remain. buffer.cleanup(); }; - if constexpr (Filtered) { - return extensions::FilteredSearch{search, filter}; - } else { - return search; - } } // Single Search @@ -641,7 +660,7 @@ class MutableVamanaIndex { ); } - template + template void search( QueryResultView results, const Queries& queries, @@ -653,31 +672,50 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - // Legalize search buffer for this search. - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - greedy_search_closure(prefetch_parameters, cancel, filter), - *this, - cancel - ); + auto body = [&](std::bool_constant) { + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + // Legalize search buffer for this search. + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + auto run = [&](const auto& search) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); + }; + auto search = greedy_search_closure( + prefetch_parameters, cancel, filter + ); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter}); + } else { + run(search); + } + }; + if (filter != nullptr) { + [&]() __attribute__((noinline)) { body(std::true_type{}); } + (); + } else { + body(std::false_type{}); + } } ); diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index f086dbe88..4aa705c08 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -57,8 +57,8 @@ class DynamicVamanaInterface : public VamanaInterface { // Check if the index has been consolidated - no `Deleted` entries remain. virtual bool is_consolidated() const = 0; - // Filtered search: only ids with filter(id) == true are returned. Keep last (ABI). - virtual void filtered_search( + // Filtered search; false if the data type does not support it. Keep last (ABI). + virtual bool filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, @@ -118,7 +118,7 @@ class DynamicVamanaImpl : public VamanaImpl results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, @@ -128,7 +128,7 @@ class DynamicVamanaImpl : public VamanaImpl(lib::Type SVS_UNUSED(type)) { - impl().template search( + impl().search( results, data::ConstSimpleDataView(queries), search_parameters, @@ -137,13 +137,10 @@ class DynamicVamanaImpl : public VamanaImpl { /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. template - void filtered_search( + bool filtered_search( QueryResultView results, data::ConstSimpleDataView queries, const VamanaSearchParameters& search_parameters, const std::function& filter ) { - impl_->filtered_search( + return impl_->filtered_search( results, AnonymousArray<2>(queries), search_parameters, filter ); } From f987b133b90e09dcfe8ec28a373b610565f86669 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 5 Nov 2026 15:40:39 -0800 Subject: [PATCH 33/47] Shorten comments and drop a redundant template argument --- include/svs/concurrent/dynamic_index.h | 8 +++----- include/svs/concurrent/translation.h | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index d9d97ae36..c934c62cf 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -185,8 +185,7 @@ template class ValidBuilder { // marks this slot as deleted. if constexpr (Filtered) { if (!invalid) { - // Lock-free read of the internal->external table. A slot whose id was - // erased has no external id and fails the filter. + // Lock-free read; an erased slot has no external id and fails. size_t external = translator_->get_external_or(i, no_external_id); invalid = external == no_external_id || !(*filter_)(external); } @@ -986,13 +985,12 @@ class MutableVamanaIndex { scratch.buffer, scratch.scratch, query, - greedy_search_closure(scratch.prefetch_parameters, cancel), + greedy_search_closure(scratch.prefetch_parameters, cancel), *this ); } - // `filter` runs on every search thread without the translator lock; it must be - // thread-safe. + // `filter` is called from every search thread without locks; it must be thread-safe. template void search( QueryResultView results, diff --git a/include/svs/concurrent/translation.h b/include/svs/concurrent/translation.h index 652c2c7b7..1a5a97544 100644 --- a/include/svs/concurrent/translation.h +++ b/include/svs/concurrent/translation.h @@ -524,7 +524,7 @@ class IDTranslator { tsl::robin_map external_to_internal_{}; // TODO: moves to #447 (same vector-based internal-to-external table); not in this PR. - // Grow-stable: `load_external` is safe without a lock. + // Segments never move when it grows, so `load_external` needs no lock. lib::SegmentedVector internal_to_external_{}; external_id_type load_external(internal_id_type i) const { From 0bc51cf458bd497d092a5f5c6b45fe6da4199a41 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 6 Nov 2026 14:52:22 -0800 Subject: [PATCH 34/47] Address review: padding and FilteredValidBuilder --- include/svs/concurrent/dynamic_index.h | 104 ++++++++++++++--------- include/svs/index/vamana/dynamic_index.h | 90 ++++++++++++-------- include/svs/index/vamana/extensions.h | 19 ++--- 3 files changed, 125 insertions(+), 88 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index fd4b0f641..ddef1ef89 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -148,17 +148,40 @@ enum class ReplaceExternalIdResult : uint8_t { inline constexpr size_t no_external_id = type_traits::sentinel_v, std::less<>>.id(); -// Template so the compiler builds a separate regular-search copy with no filter check. -template class ValidBuilder { +class ValidBuilder { public: - ValidBuilder( + ValidBuilder(const lib::SegmentedVector& status) + : status_{status} {} + + template + constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + // A neighbor is returnable only if its slot is Valid. Deleted slots + // must be skipped; Pending slots are reserved by an in-flight add and + // their vectors/edges are not yet fully published. Empty slots should + // never be reached via a valid edge, but we defend anyway. + bool invalid = + std::atomic_ref(const_cast(getindex(status_, i))) + .load(std::memory_order_acquire) != SlotMetadata::Valid; + // This neighbor should be skipped if the metadata corresponding to the given index + // marks this slot as deleted. + return PredicatedSearchNeighbor(i, distance, !invalid); + } + + private: + const lib::SegmentedVector& status_; +}; + +// Separate template so the regular search compiles without any filter check. +template class FilteredValidBuilder { + public: + FilteredValidBuilder( const lib::SegmentedVector& status, - const IDTranslator* translator = nullptr, - const std::function* filter = nullptr + const IDTranslator& translator, + const Filter& filter ) : status_{status} - , translator_{translator} - , filter_{filter} {} + , translator_{&translator} + , filter_{&filter} {} // Once the window is full, only candidates closer than its worst entry are filtered. template void skip_if_dropped_by(const Buffer& buffer) { @@ -170,26 +193,16 @@ template class ValidBuilder { template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if constexpr (Filtered) { - if (skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } + if (skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); } - // A neighbor is returnable only if its slot is Valid. Deleted slots - // must be skipped; Pending slots are reserved by an in-flight add and - // their vectors/edges are not yet fully published. Empty slots should - // never be reached via a valid edge, but we defend anyway. bool invalid = std::atomic_ref(const_cast(getindex(status_, i))) .load(std::memory_order_acquire) != SlotMetadata::Valid; - // This neighbor should be skipped if the metadata corresponding to the given index - // marks this slot as deleted. - if constexpr (Filtered) { - if (!invalid) { - // Lock-free read; an erased slot has no external id and fails. - size_t external = translator_->get_external_or(i, no_external_id); - invalid = external == no_external_id || !(*filter_)(external); - } + if (!invalid) { + // Lock-free read; an erased slot has no external id and fails. + size_t external = translator_->get_external_or(i, no_external_id); + invalid = external == no_external_id || !(*filter_)(external); } return PredicatedSearchNeighbor(i, distance, !invalid); } @@ -197,7 +210,7 @@ template class ValidBuilder { private: const lib::SegmentedVector& status_; const IDTranslator* translator_; - const std::function* filter_; + const Filter* filter_; const void* skip_buffer_ = nullptr; bool (*skip_)(const void*, float) = nullptr; }; @@ -914,10 +927,11 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - template - ValidBuilder - internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &translator_, filter}; + ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + + template + FilteredValidBuilder internal_search_builder(const Filter& filter) const { + return FilteredValidBuilder{status_, translator_, filter}; } /// @brief RAII shared lock guarding data_/graph_ against compact()'s shrink @@ -952,23 +966,29 @@ class MutableVamanaIndex { buffer.clear(); return; } - auto builder = internal_search_builder(filter); + // Perform the greedy search using the provided resources. + auto run = [&](auto& builder) { + concurrent::greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{std::span{&seed, 1}}, + builder, + prefetch_parameters, + cancel + ); + }; if constexpr (Filtered) { + auto builder = internal_search_builder(*filter); builder.skip_if_dropped_by(buffer); + run(builder); + } else { + auto builder = internal_search_builder(); + run(builder); } - // Perform the greedy search using the provided resources. - concurrent::greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{std::span{&seed, 1}}, - builder, - prefetch_parameters, - cancel - ); // Take a pass over the search buffer to remove any deleted elements that // might remain. buffer.cleanup(); diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index c14bdddef..9087d9a0c 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -92,17 +92,34 @@ inline constexpr std::string_view name(SlotMetadata metadata) { } // clang-format on -// Template so the compiler builds a separate regular-search copy with no filter check. -template class ValidBuilder { +class ValidBuilder { public: - ValidBuilder( + ValidBuilder(const std::vector& status) + : status_{status} {} + + template + constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + bool invalid = getindex(status_, i) == SlotMetadata::Deleted; + // This neighbor should be skipped if the metadata corresponding to the given index + // marks this slot as deleted. + return PredicatedSearchNeighbor(i, distance, !invalid); + } + + private: + const std::vector& status_; +}; + +// Separate template so the regular search compiles without any filter check. +template class FilteredValidBuilder { + public: + FilteredValidBuilder( const std::vector& status, - const IDTranslator* translator = nullptr, - const std::function* filter = nullptr + const IDTranslator& translator, + const Filter& filter ) : status_{status} - , translator_{translator} - , filter_{filter} {} + , translator_{&translator} + , filter_{&filter} {} // Once the window is full, only candidates closer than its worst entry are filtered. template void skip_if_dropped_by(const Buffer& buffer) { @@ -114,18 +131,12 @@ template class ValidBuilder { template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if constexpr (Filtered) { - if (skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } + if (skip_ != nullptr && skip_(skip_buffer_, distance)) { + return PredicatedSearchNeighbor(i, distance, false); } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; - // This neighbor should be skipped if the metadata corresponding to the given index - // marks this slot as deleted. - if constexpr (Filtered) { - if (!invalid) { - invalid = !(*filter_)(translator_->get_external(i)); - } + if (!invalid) { + invalid = !(*filter_)(translator_->get_external(i)); } return PredicatedSearchNeighbor(i, distance, !invalid); } @@ -133,7 +144,7 @@ template class ValidBuilder { private: const std::vector& status_; const IDTranslator* translator_; - const std::function* filter_; + const Filter* filter_; const void* skip_buffer_ = nullptr; bool (*skip_)(const void*, float) = nullptr; }; @@ -709,10 +720,11 @@ class MutableVamanaIndex { // Return a `greedy_search` compatible builder for this index. // This is an internal method, mostly used to help implement the batch iterator. - template - ValidBuilder - internal_search_builder(const std::function* filter = nullptr) const { - return ValidBuilder{status_, &translator_, filter}; + ValidBuilder internal_search_builder() const { return ValidBuilder{status_}; } + + template + FilteredValidBuilder internal_search_builder(const Filter& filter) const { + return FilteredValidBuilder{status_, translator_, filter}; } template @@ -724,23 +736,29 @@ class MutableVamanaIndex { return [&, prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { - auto builder = internal_search_builder(filter); + // Perform the greedy search using the provided resources. + auto run = [&](auto& builder) { + greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + builder, + prefetch_parameters, + cancel + ); + }; if constexpr (Filtered) { + auto builder = internal_search_builder(*filter); builder.skip_if_dropped_by(buffer); + run(builder); + } else { + auto builder = internal_search_builder(); + run(builder); } - // Perform the greedy search using the provided resources. - greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - builder, - prefetch_parameters, - cancel - ); // Take a pass over the search buffer to remove any deleted elements that // might remain. buffer.cleanup(); diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index a745a7401..b7c3983a5 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -607,18 +607,17 @@ void svs_invoke( ); // Copy back results. - for (size_t j = 0; j < num_neighbors; ++j) { - if constexpr (requires { search.filter; }) { - if (j >= search_buffer.size()) { // Filtered: fewer than k passed. - result.index(i, j) = static_cast(-1); - result.distance(i, j) = distance::comparator(distance)(0.0f, 1.0f) - ? std::numeric_limits::infinity() - : -std::numeric_limits::infinity(); - continue; - } - } + auto num_results = std::min(num_neighbors, search_buffer.size()); + for (size_t j = 0; j < num_results; ++j) { result.set(search_buffer[j], i, j); } + // Fewer than k results (e.g. a selective filter): pad with invalid ids. + for (size_t j = num_results; j < num_neighbors; ++j) { + result.index(i, j) = static_cast(-1); + result.distance(i, j) = distance::comparator(distance)(0.0f, 1.0f) + ? std::numeric_limits::infinity() + : -std::numeric_limits::infinity(); + } } } From cd561bdd8a7afe6310d66d1f29c8ddca6f13e80c Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 6 Nov 2026 15:22:58 -0800 Subject: [PATCH 35/47] Filtered search on the static Vamana index --- include/svs/index/vamana/index.h | 168 +++++++++++++++------ include/svs/orchestrators/dynamic_vamana.h | 33 ---- include/svs/orchestrators/vamana.h | 48 ++++++ 3 files changed, 168 insertions(+), 81 deletions(-) diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index 4d4a47f09..bd4d73478 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -24,6 +24,7 @@ #include "svs/core/query_result.h" #include "svs/core/recall.h" #include "svs/index/vamana/calibrate.h" +#include "svs/index/vamana/dynamic_search_buffer.h" #include "svs/index/vamana/extensions.h" #include "svs/index/vamana/greedy_search.h" #include "svs/index/vamana/search_buffer.h" @@ -39,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +52,33 @@ namespace svs::index::vamana { template class BatchIterator; +// Marks candidates failing the filter invalid. +template class FilteredNeighborBuilder { + public: + explicit FilteredNeighborBuilder(const Filter& filter) + : filter_{&filter} {} + + // Once the window is full, only candidates closer than its worst entry are filtered. + template void skip_if_dropped_by(const Buffer& buffer) { + skip_buffer_ = &buffer; + skip_ = [](const void* b, float d) { + return static_cast(b)->can_skip(d); + }; + } + + template + constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + bool valid = !(skip_ != nullptr && skip_(skip_buffer_, distance)) && + (*filter_)(static_cast(i)); + return PredicatedSearchNeighbor(i, distance, valid); + } + + private: + const Filter* filter_; + const void* skip_buffer_ = nullptr; + bool (*skip_)(const void*, float) = nullptr; +}; + struct VamanaIndexParameters { // Members public: @@ -485,25 +514,40 @@ class VamanaIndex { // This is an internal method, mostly used to help implement the batch iterator. static NeighborBuilder internal_search_builder() { return NeighborBuilder(); } + template auto greedy_search_closure( GreedySearchPrefetchParameters prefetch_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) const { - return [&, prefetch_parameters]( - const auto& query, auto& accessor, auto& distance, auto& buffer - ) { - greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - internal_search_builder(), + return [&, prefetch_parameters, - cancel - ); + filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + auto run = [&](auto& builder) { + greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + builder, + prefetch_parameters, + cancel + ); + }; + if constexpr (Filtered) { + auto builder = + FilteredNeighborBuilder>{*filter}; + builder.skip_if_dropped_by(buffer); + run(builder); + // Remove the invalid entries that remain in the buffer. + buffer.cleanup(); + } else { + auto builder = internal_search_builder(); + run(builder); + } }; } @@ -578,44 +622,72 @@ class VamanaIndex { QueryResultView result, const Queries& queries, const search_parameters_type& search_parameters, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) { threads::parallel_for( threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - // The number of neighbors to store in the result. - size_t num_neighbors = result.n_neighbors(); - - // Allocate scratchspace according to the provided search parameters. - auto search_buffer = search_buffer_type{ - // Increase the search window size if the defaults are not suitable for - // the requested number of neighbors. - search_parameters.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : search_parameters.buffer_config_, - distance::comparator(distance_), - search_parameters.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - search_parameters.prefetch_lookahead_, - search_parameters.prefetch_step_}; - - // Pre-allocate scratch space needed by the dataset implementation. - auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - - // Perform a search over the batch of queries. - extensions::per_thread_batch_search( - data_, - search_buffer, - scratch, - queries, - result, - threads::UnitRange{is}, - greedy_search_closure(prefetch_parameters, cancel), - *this, - cancel - ); + auto body = [&](std::bool_constant) { + // Filtered search needs a buffer that can hold invalid entries. + using buffer_type = std::conditional_t< + Filtered, + MutableBuffer>, + search_buffer_type>; + + // The number of neighbors to store in the result. + size_t num_neighbors = result.n_neighbors(); + + // Allocate scratchspace according to the provided search parameters. + auto search_buffer = buffer_type{ + // Increase the search window size if the defaults are not + // suitable for the requested number of neighbors. + search_parameters.buffer_config_.get_total_capacity() < + num_neighbors + ? SearchBufferConfig{num_neighbors} + : search_parameters.buffer_config_, + distance::comparator(distance_), + search_parameters.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + search_parameters.prefetch_lookahead_, + search_parameters.prefetch_step_}; + + // Pre-allocate scratch space needed by the dataset implementation. + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + + auto run = [&](const auto& search) { + // Perform a search over the batch of queries. + extensions::per_thread_batch_search( + data_, + search_buffer, + scratch, + queries, + result, + threads::UnitRange{is}, + search, + *this, + cancel + ); + }; + auto search = greedy_search_closure( + prefetch_parameters, cancel, filter + ); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter}); + } else { + run(search); + } + }; + // Keep the filtered body out of line so regular search is unaffected. + if (filter != nullptr) { + [&]() __attribute__((noinline)) { body(std::true_type{}); } + (); + } else { + body(std::false_type{}); + } } ); } diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 6ace9b035..339974899 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -56,14 +56,6 @@ class DynamicVamanaInterface : public VamanaInterface { // Check if the index has been consolidated - no `Deleted` entries remain. virtual bool is_consolidated() const = 0; - - // Filtered search; false if the data type does not support it. Keep last (ABI). - virtual bool filtered_search( - svs::QueryResultView results, - AnonymousArray<2> queries, - const index::vamana::VamanaSearchParameters& search_parameters, - const std::function& filter - ) = 0; }; template @@ -117,31 +109,6 @@ class DynamicVamanaImpl : public VamanaImpl results, - AnonymousArray<2> queries, - const index::vamana::VamanaSearchParameters& search_parameters, - const std::function& filter - ) override { - lib::match( - QueryTypes{}, - queries.type(), - [&](lib::Type SVS_UNUSED(type)) { - impl().search( - results, - data::ConstSimpleDataView(queries), - search_parameters, - lib::Returns(lib::Const()), - &filter - ); - }, - [&](svs::DataType data_type) { - throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); - } - ); - return true; - } }; // Forward Declaractions. diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index 5787e8a7a..5bd633a50 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -35,6 +35,7 @@ // stdlib #include +#include #include #include #include @@ -108,6 +109,14 @@ class VamanaInterface { // through the wrong slot with no link-time diagnostic. virtual svs::index::vamana::MemoryBreakdown get_memory_breakdown() const = 0; virtual svs::index::vamana::VamanaIndexParameters parameters() const = 0; + + // Filtered search; keep last (ABI). + virtual bool filtered_search( + svs::QueryResultView results, + AnonymousArray<2> queries, + const index::vamana::VamanaSearchParameters& search_parameters, + const std::function& filter + ) = 0; }; template @@ -283,6 +292,31 @@ class VamanaImpl : public manager::ManagerImpl { svs::index::vamana::VamanaIndexParameters parameters() const override { return impl().parameters(); } + + bool filtered_search( + svs::QueryResultView results, + AnonymousArray<2> queries, + const index::vamana::VamanaSearchParameters& search_parameters, + const std::function& filter + ) override { + lib::match( + QueryTypes{}, + queries.type(), + [&](lib::Type SVS_UNUSED(type)) { + impl().search( + results, + data::ConstSimpleDataView(queries), + search_parameters, + lib::Returns(lib::Const()), + &filter + ); + }, + [&](svs::DataType data_type) { + throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); + } + ); + return true; + } }; ///// Forward declarations @@ -776,6 +810,20 @@ class Vamana : public manager::IndexManager { return impl_->parameters(); } + /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. + /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. + template + bool filtered_search( + QueryResultView results, + data::ConstSimpleDataView queries, + const VamanaSearchParameters& search_parameters, + const std::function& filter + ) { + return impl_->filtered_search( + results, AnonymousArray<2>(queries), search_parameters, filter + ); + } + private: template < manager::QueryTypeDefinition QueryTypes, From f6cc27a5f3cce9a0dd57a52a1d4e820ebcaeac24 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 6 Nov 2026 17:14:51 -0800 Subject: [PATCH 36/47] Fix unused lambda capture in regular search --- include/svs/index/vamana/dynamic_index.h | 1 + include/svs/index/vamana/index.h | 1 + 2 files changed, 2 insertions(+) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 9087d9a0c..d4605f268 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -736,6 +736,7 @@ class MutableVamanaIndex { return [&, prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + static_cast(filter); // Unused in regular search. // Perform the greedy search using the provided resources. auto run = [&](auto& builder) { greedy_search( diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index bd4d73478..d5c87d9a6 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -523,6 +523,7 @@ class VamanaIndex { return [&, prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + static_cast(filter); // Unused in regular search. auto run = [&](auto& builder) { greedy_search( graph_, From 0652eebd933f91fc7db532bbe1cee0d23c6b1332 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Fri, 6 Nov 2026 17:56:54 -0800 Subject: [PATCH 37/47] Fix unused lambda capture in concurrent regular search --- include/svs/concurrent/dynamic_index.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index ddef1ef89..6328a08a3 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -961,6 +961,7 @@ class MutableVamanaIndex { return [&, prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { + static_cast(filter); // Unused in regular search. const Idx seed = entry_point(); if (seed == NO_ENTRY) { buffer.clear(); From 372dcd755d5690d052fa702b979ed50c764c310e Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 11:04:31 -0800 Subject: [PATCH 38/47] Note the ABI effect of the new VamanaInterface virtual --- include/svs/orchestrators/vamana.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index 5bd633a50..1db9ea1c3 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -110,7 +110,7 @@ class VamanaInterface { virtual svs::index::vamana::MemoryBreakdown get_memory_breakdown() const = 0; virtual svs::index::vamana::VamanaIndexParameters parameters() const = 0; - // Filtered search; keep last (ABI). + // Appended last; shifts DynamicVamanaInterface's virtual slots (ABI change). virtual bool filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, From 8eb7a02fe6300be0979332f497730dd27b0a8e0b Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 11:05:11 -0800 Subject: [PATCH 39/47] Revert "Note the ABI effect of the new VamanaInterface virtual" This reverts commit 372dcd755d5690d052fa702b979ed50c764c310e. --- include/svs/orchestrators/vamana.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index 1db9ea1c3..5bd633a50 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -110,7 +110,7 @@ class VamanaInterface { virtual svs::index::vamana::MemoryBreakdown get_memory_breakdown() const = 0; virtual svs::index::vamana::VamanaIndexParameters parameters() const = 0; - // Appended last; shifts DynamicVamanaInterface's virtual slots (ABI change). + // Filtered search; keep last (ABI). virtual bool filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, From f19ceb8ffcd33caef566aff94ccf7d02611d7d89 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 14:35:21 -0800 Subject: [PATCH 40/47] Pass the filter to the search top-up explicitly Drop the FilteredSearch closure wrapper: single_search and per_thread_batch_search take the filter as a trailing argument and hand it to the buffer top-up. A filtered concurrent search re-checks the filter on the external ids it returns, and the filter contract is documented. --- include/svs/concurrent/dynamic_index.h | 38 +++++++++++++------- include/svs/index/vamana/dynamic_index.h | 9 ++--- include/svs/index/vamana/extensions.h | 42 +++++++++++----------- include/svs/index/vamana/index.h | 9 ++--- include/svs/orchestrators/dynamic_vamana.h | 1 + include/svs/orchestrators/vamana.h | 1 + 6 files changed, 55 insertions(+), 45 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 6328a08a3..861e1d32b 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -1017,7 +1017,8 @@ class MutableVamanaIndex { ); } - // `filter` is called from every search thread without locks; it must be thread-safe. + // `filter` is called from every search thread without locks; it must be thread-safe + // and must not call into this index. template void search( QueryResultView results, @@ -1060,18 +1061,14 @@ class MutableVamanaIndex { threads::UnitRange{is}, search, *this, - cancel + cancel, + filter ); }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter} - ); - } else { - run(search); - } + run(search); }; if (filter != nullptr) { [&]() __attribute__((noinline)) { body(std::true_type{}); } @@ -1091,6 +1088,24 @@ class MutableVamanaIndex { // After the search procedure, the indices in `results` are internal. // Perform one more pass to convert these to external ids. translate_to_external(results.indices()); + + if (filter != nullptr) { + // A concurrent remap may have changed the answer: only return ids that pass + // now. + const float worst = distance::comparator(distance_)(0.0f, 1.0f) + ? std::numeric_limits::infinity() + : -std::numeric_limits::infinity(); + for (size_t i = 0; i < results.n_queries(); ++i) { + for (size_t j = 0; j < results.n_neighbors(); ++j) { + auto id = results.index(i, j); + if (id != static_cast(-1) && + (id == no_external_id || !(*filter)(id))) { + results.index(i, j) = static_cast(-1); + results.distance(i, j) = worst; + } + } + } + } } /// @@ -2227,7 +2242,8 @@ SVS_FORCE_INLINE void svs_invoke( const Query& query, const Search& search, const MutableVamanaIndex& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) { using index_type = MutableVamanaIndex; vamana::extensions::single_search( @@ -2244,10 +2260,6 @@ SVS_FORCE_INLINE void svs_invoke( if (cancel()) { return; } - const std::function* filter = nullptr; - if constexpr (requires { search.filter; }) { - filter = search.filter; - } index.supplement_search_buffer(search_buffer, query, filter); } diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index d4605f268..ff60c3a08 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -821,17 +821,14 @@ class MutableVamanaIndex { threads::UnitRange{is}, search, *this, - cancel + cancel, + filter ); }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter}); - } else { - run(search); - } + run(search); }; if (filter != nullptr) { [&]() __attribute__((noinline)) { body(std::true_type{}); } diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index b7c3983a5..7c9545117 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -395,6 +395,7 @@ struct VamanaSingleSearchType { /// ``svs::index::vamana::extensions::single_search_setup``(). /// @param query The query used for this search. /// @param search A search functor. See the extended description. + /// @param filter Optional filter of a filtered search, used to top up the buffer. /// /// API of the ``search`` argument. /// This argument is invocable as follows: @@ -422,9 +423,12 @@ struct VamanaSingleSearchType { const Query& query, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) const { - svs::svs_invoke(*this, data, search_buffer, scratch, query, search, index, cancel); + svs::svs_invoke( + *this, data, search_buffer, scratch, query, search, index, cancel, filter + ); } }; @@ -458,15 +462,6 @@ void check_and_supplement_search_buffer( } } -/// A search closure that also carries the filter of a filtered search. -template struct FilteredSearch { - Search search; - const std::function* filter; - template void operator()(Args&&... args) const { - search(std::forward(args)...); - } -}; - /// Customization point object for processing single queries. inline constexpr VamanaSingleSearchType single_search{}; @@ -488,7 +483,8 @@ SVS_FORCE_INLINE void svs_invoke( const Query& query, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) { // Check if request to cancel the search if (cancel()) { @@ -499,10 +495,6 @@ SVS_FORCE_INLINE void svs_invoke( search(query, accessor, distance, search_buffer); if constexpr (Index::needs_id_translation) { - const std::function* filter = nullptr; - if constexpr (requires { search.filter; }) { - filter = search.filter; - } check_and_supplement_search_buffer(index, search_buffer, query, filter); } } @@ -553,7 +545,8 @@ struct VamanaPerThreadBatchSearchType { threads::UnitRange thread_indices, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) const { svs::svs_invoke( *this, @@ -565,7 +558,8 @@ struct VamanaPerThreadBatchSearchType { thread_indices, search, index, - cancel + cancel, + filter ); } }; @@ -592,7 +586,8 @@ void svs_invoke( threads::UnitRange thread_indices, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), + const std::function* filter = nullptr ) { // Fallback implementation size_t num_neighbors = result.n_neighbors(); @@ -603,7 +598,14 @@ void svs_invoke( } // Perform search - results will be queued in the search buffer. single_search( - dataset, search_buffer, distance, queries.get_datum(i), search, index, cancel + dataset, + search_buffer, + distance, + queries.get_datum(i), + search, + index, + cancel, + filter ); // Copy back results. diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index d5c87d9a6..92a3d3b94 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -670,17 +670,14 @@ class VamanaIndex { threads::UnitRange{is}, search, *this, - cancel + cancel, + filter ); }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter}); - } else { - run(search); - } + run(search); }; // Keep the filtered body out of line so regular search is unaffected. if (filter != nullptr) { diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index 339974899..e256c42e2 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -612,6 +612,7 @@ class DynamicVamana : public manager::IndexManager { /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. + /// ``filter`` must be thread-safe and must not call into the same index. template bool filtered_search( QueryResultView results, diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index 5bd633a50..2637e7f58 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -812,6 +812,7 @@ class Vamana : public manager::IndexManager { /// @brief Filtered search: only ids for which ``filter(id)`` is true are returned. /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. + /// ``filter`` must be thread-safe and must not call into the same index. template bool filtered_search( QueryResultView results, From 4cbd1fd327209413b35c6854fa644c564d837eba Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 15:24:38 -0800 Subject: [PATCH 41/47] Keep FilteredSearch for now --- include/svs/concurrent/dynamic_index.h | 17 +++++++--- include/svs/index/vamana/dynamic_index.h | 9 +++-- include/svs/index/vamana/extensions.h | 42 +++++++++++------------- include/svs/index/vamana/index.h | 9 +++-- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 861e1d32b..56c9aa0f5 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -1061,14 +1061,18 @@ class MutableVamanaIndex { threads::UnitRange{is}, search, *this, - cancel, - filter + cancel ); }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - run(search); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter} + ); + } else { + run(search); + } }; if (filter != nullptr) { [&]() __attribute__((noinline)) { body(std::true_type{}); } @@ -2242,8 +2246,7 @@ SVS_FORCE_INLINE void svs_invoke( const Query& query, const Search& search, const MutableVamanaIndex& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* filter = nullptr + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) { using index_type = MutableVamanaIndex; vamana::extensions::single_search( @@ -2260,6 +2263,10 @@ SVS_FORCE_INLINE void svs_invoke( if (cancel()) { return; } + const std::function* filter = nullptr; + if constexpr (requires { search.filter; }) { + filter = search.filter; + } index.supplement_search_buffer(search_buffer, query, filter); } diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index ff60c3a08..d4605f268 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -821,14 +821,17 @@ class MutableVamanaIndex { threads::UnitRange{is}, search, *this, - cancel, - filter + cancel ); }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - run(search); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter}); + } else { + run(search); + } }; if (filter != nullptr) { [&]() __attribute__((noinline)) { body(std::true_type{}); } diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 7c9545117..b7c3983a5 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -395,7 +395,6 @@ struct VamanaSingleSearchType { /// ``svs::index::vamana::extensions::single_search_setup``(). /// @param query The query used for this search. /// @param search A search functor. See the extended description. - /// @param filter Optional filter of a filtered search, used to top up the buffer. /// /// API of the ``search`` argument. /// This argument is invocable as follows: @@ -423,12 +422,9 @@ struct VamanaSingleSearchType { const Query& query, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* filter = nullptr + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) const { - svs::svs_invoke( - *this, data, search_buffer, scratch, query, search, index, cancel, filter - ); + svs::svs_invoke(*this, data, search_buffer, scratch, query, search, index, cancel); } }; @@ -462,6 +458,15 @@ void check_and_supplement_search_buffer( } } +/// A search closure that also carries the filter of a filtered search. +template struct FilteredSearch { + Search search; + const std::function* filter; + template void operator()(Args&&... args) const { + search(std::forward(args)...); + } +}; + /// Customization point object for processing single queries. inline constexpr VamanaSingleSearchType single_search{}; @@ -483,8 +488,7 @@ SVS_FORCE_INLINE void svs_invoke( const Query& query, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* filter = nullptr + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) { // Check if request to cancel the search if (cancel()) { @@ -495,6 +499,10 @@ SVS_FORCE_INLINE void svs_invoke( search(query, accessor, distance, search_buffer); if constexpr (Index::needs_id_translation) { + const std::function* filter = nullptr; + if constexpr (requires { search.filter; }) { + filter = search.filter; + } check_and_supplement_search_buffer(index, search_buffer, query, filter); } } @@ -545,8 +553,7 @@ struct VamanaPerThreadBatchSearchType { threads::UnitRange thread_indices, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* filter = nullptr + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) const { svs::svs_invoke( *this, @@ -558,8 +565,7 @@ struct VamanaPerThreadBatchSearchType { thread_indices, search, index, - cancel, - filter + cancel ); } }; @@ -586,8 +592,7 @@ void svs_invoke( threads::UnitRange thread_indices, const Search& search, const Index& index, - const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()), - const std::function* filter = nullptr + const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) { // Fallback implementation size_t num_neighbors = result.n_neighbors(); @@ -598,14 +603,7 @@ void svs_invoke( } // Perform search - results will be queued in the search buffer. single_search( - dataset, - search_buffer, - distance, - queries.get_datum(i), - search, - index, - cancel, - filter + dataset, search_buffer, distance, queries.get_datum(i), search, index, cancel ); // Copy back results. diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index 92a3d3b94..d5c87d9a6 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -670,14 +670,17 @@ class VamanaIndex { threads::UnitRange{is}, search, *this, - cancel, - filter + cancel ); }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - run(search); + if constexpr (Filtered) { + run(extensions::FilteredSearch{search, filter}); + } else { + run(search); + } }; // Keep the filtered body out of line so regular search is unaffected. if (filter != nullptr) { From 85c297d2082a4f003e31520d60a1dccb2754e535 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 16:15:11 -0800 Subject: [PATCH 42/47] Do not apply the filter in the search top-up yet --- include/svs/concurrent/dynamic_index.h | 36 ++---------------------- include/svs/index/vamana/dynamic_index.h | 2 +- include/svs/index/vamana/extensions.h | 24 ++-------------- include/svs/index/vamana/index.h | 2 +- 4 files changed, 7 insertions(+), 57 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 56c9aa0f5..76f4c705c 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -1068,8 +1068,7 @@ class MutableVamanaIndex { prefetch_parameters, cancel, filter ); if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter} - ); + run(search); } else { run(search); } @@ -2135,38 +2134,13 @@ class MutableVamanaIndex { /// Called from the ``extensions::single_search`` override below. /// template - void supplement_search_buffer( - SearchBuffer& search_buffer, - const Query& query, - const std::function* filter = nullptr - ) const { + void supplement_search_buffer(SearchBuffer& search_buffer, const Query& query) const { if (search_buffer.valid() >= search_buffer.target_window() || search_buffer.valid() >= size()) { return; } search_buffer.sort(); - if (filter != nullptr) { - // compact_mutex_ shared (held by the caller) keeps the slots in place. - auto builder = internal_search_builder(); - for (size_t i = 0, n = status_.size(); i < n; ++i) { - if (std::atomic_ref(const_cast(status_[i])) - .load(std::memory_order_acquire) != SlotMetadata::Valid) { - continue; - } - auto external = unsafe_translate_internal_id(i); - if (external == no_external_id || !(*filter)(external)) { - continue; - } - auto dist = extensions::get_distance_ext(data_, distance_, i, query); - search_buffer.insert(builder(static_cast(i), dist)); - if (search_buffer.valid() >= search_buffer.target_window()) { - break; - } - } - return; - } - // translator_mutex_ shared: guards both the traversal of ``translator_`` and the // internal IDs it yields against erasure by consolidate()/compact(). std::shared_lock translator_lock{*translator_mutex_}; @@ -2263,11 +2237,7 @@ SVS_FORCE_INLINE void svs_invoke( if (cancel()) { return; } - const std::function* filter = nullptr; - if constexpr (requires { search.filter; }) { - filter = search.filter; - } - index.supplement_search_buffer(search_buffer, query, filter); + index.supplement_search_buffer(search_buffer, query); } ///// Deduction Guides. diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index d4605f268..298d04be6 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -828,7 +828,7 @@ class MutableVamanaIndex { prefetch_parameters, cancel, filter ); if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter}); + run(search); } else { run(search); } diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index b7c3983a5..b13e43f21 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -27,7 +27,6 @@ #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" -#include #include namespace svs::index::vamana::extensions { @@ -435,18 +434,12 @@ struct VamanaSingleSearchType { /// with additional results. template void check_and_supplement_search_buffer( - const Index& index, - SearchBuffer& search_buffer, - const Query& query, - const std::function* filter = nullptr + const Index& index, SearchBuffer& search_buffer, const Query& query ) { if (search_buffer.valid() < search_buffer.target_window() && search_buffer.valid() < index.size()) { search_buffer.sort(); for (auto external_id : index.external_ids()) { - if (filter != nullptr && !(*filter)(external_id)) { - continue; - } auto internal_id = index.translate_external_id(external_id); auto dist = index.get_distance(external_id, query); auto builder = index.internal_search_builder(); @@ -458,15 +451,6 @@ void check_and_supplement_search_buffer( } } -/// A search closure that also carries the filter of a filtered search. -template struct FilteredSearch { - Search search; - const std::function* filter; - template void operator()(Args&&... args) const { - search(std::forward(args)...); - } -}; - /// Customization point object for processing single queries. inline constexpr VamanaSingleSearchType single_search{}; @@ -499,11 +483,7 @@ SVS_FORCE_INLINE void svs_invoke( search(query, accessor, distance, search_buffer); if constexpr (Index::needs_id_translation) { - const std::function* filter = nullptr; - if constexpr (requires { search.filter; }) { - filter = search.filter; - } - check_and_supplement_search_buffer(index, search_buffer, query, filter); + check_and_supplement_search_buffer(index, search_buffer, query); } } diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index d5c87d9a6..390235b52 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -677,7 +677,7 @@ class VamanaIndex { prefetch_parameters, cancel, filter ); if constexpr (Filtered) { - run(extensions::FilteredSearch{search, filter}); + run(search); } else { run(search); } From f886b625df6e8633c858f9639c076e3b1b26b772 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 16:54:02 -0800 Subject: [PATCH 43/47] Skip candidates a full buffer would drop in greedy_search --- include/svs/concurrent/dynamic_index.h | 14 -------------- include/svs/concurrent/greedy_search.h | 4 +++- include/svs/index/vamana/dynamic_index.h | 14 -------------- include/svs/index/vamana/greedy_search.h | 4 +++- include/svs/index/vamana/index.h | 14 +------------- 5 files changed, 7 insertions(+), 43 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 76f4c705c..2470c71b6 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -183,19 +183,8 @@ template class FilteredValidBuilder { , translator_{&translator} , filter_{&filter} {} - // Once the window is full, only candidates closer than its worst entry are filtered. - template void skip_if_dropped_by(const Buffer& buffer) { - skip_buffer_ = &buffer; - skip_ = [](const void* b, float d) { - return static_cast(b)->can_skip(d); - }; - } - template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if (skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } bool invalid = std::atomic_ref(const_cast(getindex(status_, i))) .load(std::memory_order_acquire) != SlotMetadata::Valid; @@ -211,8 +200,6 @@ template class FilteredValidBuilder { const lib::SegmentedVector& status_; const IDTranslator* translator_; const Filter* filter_; - const void* skip_buffer_ = nullptr; - bool (*skip_)(const void*, float) = nullptr; }; template @@ -984,7 +971,6 @@ class MutableVamanaIndex { }; if constexpr (Filtered) { auto builder = internal_search_builder(*filter); - builder.skip_if_dropped_by(buffer); run(builder); } else { auto builder = internal_search_builder(); diff --git a/include/svs/concurrent/greedy_search.h b/include/svs/concurrent/greedy_search.h index fe8d7fd59..0b0864f9e 100644 --- a/include/svs/concurrent/greedy_search.h +++ b/include/svs/concurrent/greedy_search.h @@ -146,7 +146,9 @@ void greedy_search( // Compute distance and update search buffer. auto dist = distance::compute(distance_function, query, accessor(dataset, id)); - search_buffer.insert(builder(id, dist)); + if (!search_buffer.can_skip(dist)) { + search_buffer.insert(builder(id, dist)); + } } // Validate that no concurrent write occurred during the read. diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 298d04be6..e83aeb3cb 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -121,19 +121,8 @@ template class FilteredValidBuilder { , translator_{&translator} , filter_{&filter} {} - // Once the window is full, only candidates closer than its worst entry are filtered. - template void skip_if_dropped_by(const Buffer& buffer) { - skip_buffer_ = &buffer; - skip_ = [](const void* b, float d) { - return static_cast(b)->can_skip(d); - }; - } - template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - if (skip_ != nullptr && skip_(skip_buffer_, distance)) { - return PredicatedSearchNeighbor(i, distance, false); - } bool invalid = getindex(status_, i) == SlotMetadata::Deleted; if (!invalid) { invalid = !(*filter_)(translator_->get_external(i)); @@ -145,8 +134,6 @@ template class FilteredValidBuilder { const std::vector& status_; const IDTranslator* translator_; const Filter* filter_; - const void* skip_buffer_ = nullptr; - bool (*skip_)(const void*, float) = nullptr; }; namespace detail { @@ -754,7 +741,6 @@ class MutableVamanaIndex { }; if constexpr (Filtered) { auto builder = internal_search_builder(*filter); - builder.skip_if_dropped_by(buffer); run(builder); } else { auto builder = internal_search_builder(); diff --git a/include/svs/index/vamana/greedy_search.h b/include/svs/index/vamana/greedy_search.h index f12c0129f..30225578f 100644 --- a/include/svs/index/vamana/greedy_search.h +++ b/include/svs/index/vamana/greedy_search.h @@ -197,7 +197,9 @@ void greedy_search( // Compute distance and update search buffer. auto dist = distance::compute(distance_function, query, accessor(dataset, id)); - search_buffer.insert(builder(id, dist)); + if (!search_buffer.can_skip(dist)) { + search_buffer.insert(builder(id, dist)); + } } } } diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index 390235b52..da4721459 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -58,25 +58,14 @@ template class FilteredNeighborBuilder { explicit FilteredNeighborBuilder(const Filter& filter) : filter_{&filter} {} - // Once the window is full, only candidates closer than its worst entry are filtered. - template void skip_if_dropped_by(const Buffer& buffer) { - skip_buffer_ = &buffer; - skip_ = [](const void* b, float d) { - return static_cast(b)->can_skip(d); - }; - } - template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - bool valid = !(skip_ != nullptr && skip_(skip_buffer_, distance)) && - (*filter_)(static_cast(i)); + bool valid = (*filter_)(static_cast(i)); return PredicatedSearchNeighbor(i, distance, valid); } private: const Filter* filter_; - const void* skip_buffer_ = nullptr; - bool (*skip_)(const void*, float) = nullptr; }; struct VamanaIndexParameters { @@ -541,7 +530,6 @@ class VamanaIndex { if constexpr (Filtered) { auto builder = FilteredNeighborBuilder>{*filter}; - builder.skip_if_dropped_by(buffer); run(builder); // Remove the invalid entries that remain in the buffer. buffer.cleanup(); From b3059361cc2a658e32a444504435160a0136c225 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Mon, 9 Nov 2026 16:54:07 -0800 Subject: [PATCH 44/47] Fail when the filter calls into the concurrent index --- include/svs/concurrent/dynamic_index.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 2470c71b6..d23590ef7 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -171,6 +171,20 @@ class ValidBuilder { const lib::SegmentedVector& status_; }; +// Set while the user's filter runs on this thread, so re-entering the index fails fast. +inline thread_local bool in_user_filter = false; + +struct FilterScope { + FilterScope() { in_user_filter = true; } + ~FilterScope() { in_user_filter = false; } +}; + +inline void check_not_in_filter() { + if (in_user_filter) { + throw ANNEXCEPTION("filter must not call into the same index"); + } +} + // Separate template so the regular search compiles without any filter check. template class FilteredValidBuilder { public: @@ -184,13 +198,14 @@ template class FilteredValidBuilder { , filter_{&filter} {} template - constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { + PredicatedSearchNeighbor operator()(I i, float distance) const { bool invalid = std::atomic_ref(const_cast(getindex(status_, i))) .load(std::memory_order_acquire) != SlotMetadata::Valid; if (!invalid) { // Lock-free read; an erased slot has no external id and fails. size_t external = translator_->get_external_or(i, no_external_id); + FilterScope scope; invalid = external == no_external_id || !(*filter_)(external); } return PredicatedSearchNeighbor(i, distance, !invalid); @@ -928,6 +943,7 @@ class MutableVamanaIndex { /// Hold this through internal-to-external ID translation. Acquire the translation /// lock inside this guard, preserving the compact -> translator lock order. [[nodiscard]] std::shared_lock lock_storage_shared() const { + check_not_in_filter(); std::shared_lock gatekeeper_lock{*compact_gatekeeper_}; return std::shared_lock(*compact_mutex_); } @@ -1087,6 +1103,7 @@ class MutableVamanaIndex { for (size_t i = 0; i < results.n_queries(); ++i) { for (size_t j = 0; j < results.n_neighbors(); ++j) { auto id = results.index(i, j); + FilterScope scope; if (id != static_cast(-1) && (id == no_external_id || !(*filter)(id))) { results.index(i, j) = static_cast(-1); @@ -1490,6 +1507,7 @@ class MutableVamanaIndex { /// improve performance but requires more working memory. /// void compact(Idx batch_size = 1'000) { + check_not_in_filter(); std::lock_guard gatekeeper_lock{*compact_gatekeeper_}; std::lock_guard compact_lock{*compact_mutex_}; From b01b6fe9e6543806dc9790ff30c6f2b4f21abe54 Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 12 Nov 2026 12:34:13 -0800 Subject: [PATCH 45/47] Use only the translator in the dynamic filtered builder --- include/svs/index/vamana/dynamic_index.h | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index e83aeb3cb..582a9f18c 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -112,26 +112,19 @@ class ValidBuilder { // Separate template so the regular search compiles without any filter check. template class FilteredValidBuilder { public: - FilteredValidBuilder( - const std::vector& status, - const IDTranslator& translator, - const Filter& filter - ) - : status_{status} - , translator_{&translator} + FilteredValidBuilder(const IDTranslator& translator, const Filter& filter) + : translator_{&translator} , filter_{&filter} {} template constexpr PredicatedSearchNeighbor operator()(I i, float distance) const { - bool invalid = getindex(status_, i) == SlotMetadata::Deleted; - if (!invalid) { - invalid = !(*filter_)(translator_->get_external(i)); - } + // delete_entries clears the translator entry, so a missing id means deleted. + auto id = translator_->get_external_or(i, IDTranslator::invalid_external_id); + bool invalid = id == IDTranslator::invalid_external_id || !(*filter_)(id); return PredicatedSearchNeighbor(i, distance, !invalid); } private: - const std::vector& status_; const IDTranslator* translator_; const Filter* filter_; }; @@ -711,7 +704,7 @@ class MutableVamanaIndex { template FilteredValidBuilder internal_search_builder(const Filter& filter) const { - return FilteredValidBuilder{status_, translator_, filter}; + return FilteredValidBuilder{translator_, filter}; } template @@ -1047,12 +1040,14 @@ class MutableVamanaIndex { return ids.size(); } + private: void delete_entry(size_t i) { SlotMetadata& meta = getindex(status_, i); assert(meta == SlotMetadata::Valid); meta = SlotMetadata::Deleted; } + public: bool is_deleted(size_t i) const { return status_[i] != SlotMetadata::Valid; } Idx entry_point() const { From 6490ea9eb083b4fc98c437f5e640f529ea757c2d Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 12 Nov 2026 14:06:47 -0800 Subject: [PATCH 46/47] Address review: small fixes Reject the reserved external id, make the filter re-entry guard per index, mark translated ids that vanish as invalid, drop identical filtered/unfiltered branches, share the query-type dispatch for filtered_search (now void), and fix the metadata estimate comment. --- include/svs/concurrent/dynamic_index.h | 80 +++++++++++----------- include/svs/core/translation.h | 14 ++++ include/svs/index/vamana/dynamic_index.h | 37 ++++------ include/svs/index/vamana/index.h | 31 ++++----- include/svs/orchestrators/dynamic_vamana.h | 4 +- include/svs/orchestrators/manager.h | 49 ++++++++----- include/svs/orchestrators/vamana.h | 30 +++----- 7 files changed, 122 insertions(+), 123 deletions(-) diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index d23590ef7..48a3ccc8d 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -171,16 +171,20 @@ class ValidBuilder { const lib::SegmentedVector& status_; }; -// Set while the user's filter runs on this thread, so re-entering the index fails fast. -inline thread_local bool in_user_filter = false; +// The index whose user filter runs on this thread, so re-entering it fails fast. +inline thread_local const void* filtering_index = nullptr; struct FilterScope { - FilterScope() { in_user_filter = true; } - ~FilterScope() { in_user_filter = false; } + explicit FilterScope(const void* index) + : previous_{filtering_index} { + filtering_index = index; + } + ~FilterScope() { filtering_index = previous_; } + const void* previous_; }; -inline void check_not_in_filter() { - if (in_user_filter) { +inline void check_not_in_filter(const void* index) { + if (filtering_index == index) { throw ANNEXCEPTION("filter must not call into the same index"); } } @@ -191,11 +195,13 @@ template class FilteredValidBuilder { FilteredValidBuilder( const lib::SegmentedVector& status, const IDTranslator& translator, - const Filter& filter + const Filter& filter, + const void* owner ) : status_{status} , translator_{&translator} - , filter_{&filter} {} + , filter_{&filter} + , owner_{owner} {} template PredicatedSearchNeighbor operator()(I i, float distance) const { @@ -205,7 +211,7 @@ template class FilteredValidBuilder { if (!invalid) { // Lock-free read; an erased slot has no external id and fails. size_t external = translator_->get_external_or(i, no_external_id); - FilterScope scope; + FilterScope scope{owner_}; invalid = external == no_external_id || !(*filter_)(external); } return PredicatedSearchNeighbor(i, distance, !invalid); @@ -215,6 +221,7 @@ template class FilteredValidBuilder { const lib::SegmentedVector& status_; const IDTranslator* translator_; const Filter* filter_; + const void* owner_; }; template @@ -652,11 +659,9 @@ class MutableVamanaIndex { size_t metadata_bytes = status_.capacity() * sizeof(SlotMetadata); metadata_bytes += entry_point_.capacity() * sizeof(typename entry_point_type::value_type); - // The IDTranslator holds two tsl::robin_map instances (external->internal and - // internal->external), neither of which exposes its allocated byte count. We - // approximate the storage as the id pair held in each of the two directions. This - // ignores the maps' load-factor slack and control bytes, so it is an estimate of - // the hash-map overhead that is accurate to within a few percent. + // Estimate: the id pair per entry, counted twice (external->internal hash map and + // internal->external vector); ignores hash-map load-factor slack and vector spare + // capacity. metadata_bytes += 2 * translator_.size() * (sizeof(IDTranslator::external_id_type) + sizeof(IDTranslator::internal_id_type)); @@ -933,7 +938,7 @@ class MutableVamanaIndex { template FilteredValidBuilder internal_search_builder(const Filter& filter) const { - return FilteredValidBuilder{status_, translator_, filter}; + return FilteredValidBuilder{status_, translator_, filter, this}; } /// @brief RAII shared lock guarding data_/graph_ against compact()'s shrink @@ -943,7 +948,7 @@ class MutableVamanaIndex { /// Hold this through internal-to-external ID translation. Acquire the translation /// lock inside this guard, preserving the compact -> translator lock order. [[nodiscard]] std::shared_lock lock_storage_shared() const { - check_not_in_filter(); + check_not_in_filter(this); std::shared_lock gatekeeper_lock{*compact_gatekeeper_}; return std::shared_lock(*compact_mutex_); } @@ -1053,27 +1058,20 @@ class MutableVamanaIndex { auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - auto run = [&](const auto& search) { - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - if constexpr (Filtered) { - run(search); - } else { - run(search); - } + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); }; if (filter != nullptr) { [&]() __attribute__((noinline)) { body(std::true_type{}); } @@ -1103,10 +1101,14 @@ class MutableVamanaIndex { for (size_t i = 0; i < results.n_queries(); ++i) { for (size_t j = 0; j < results.n_neighbors(); ++j) { auto id = results.index(i, j); - FilterScope scope; - if (id != static_cast(-1) && - (id == no_external_id || !(*filter)(id))) { - results.index(i, j) = static_cast(-1); + if (id != static_cast(-1)) { + FilterScope scope{this}; + if (!(*filter)(id)) { + results.index(i, j) = static_cast(-1); + } + } + // Also covers ids a concurrent erase left without a translation. + if (results.index(i, j) == static_cast(-1)) { results.distance(i, j) = worst; } } @@ -1507,7 +1509,7 @@ class MutableVamanaIndex { /// improve performance but requires more working memory. /// void compact(Idx batch_size = 1'000) { - check_not_in_filter(); + check_not_in_filter(this); std::lock_guard gatekeeper_lock{*compact_gatekeeper_}; std::lock_guard compact_lock{*compact_mutex_}; diff --git a/include/svs/core/translation.h b/include/svs/core/translation.h index 467fcca8f..259049ae1 100644 --- a/include/svs/core/translation.h +++ b/include/svs/core/translation.h @@ -41,6 +41,7 @@ #include #include #include +#include namespace svs { @@ -121,6 +122,7 @@ class IDTranslator { if (!lib::all_unique(int_begin, int_end)) { throw ANNEXCEPTION("Internal IDs contain repeat elements!"); } + check_external_not_reserved(ext_begin, ext_end); check_external_free(ext_begin, ext_end); check_internal_free(int_begin, int_end); } @@ -180,6 +182,8 @@ class IDTranslator { throw ANNEXCEPTION("Internal IDs contain repeat elements!"); } + check_external_not_reserved(ext_begin, ext_end); + // Validation pass for (auto e = ext_begin; e != ext_end; ++e) { auto found = external_to_internal_.find(*e); @@ -374,6 +378,16 @@ class IDTranslator { /// @param begin Start forward iterator to the external IDs to check. /// @param end End forward iterator to the external IDs to check. /// + template + void check_external_not_reserved(const Begin& begin, const End& end) const { + check( + begin, + end, + [](auto e) { return std::cmp_equal(e, invalid_external_id); }, + "Reserved external" + ); + } + template void check_external_free(const Begin& begin, const End& end) const { check( diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 582a9f18c..ac7131b71 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -537,11 +537,9 @@ class MutableVamanaIndex { size_t metadata_bytes = status_.capacity() * sizeof(SlotMetadata); metadata_bytes += entry_point_.capacity() * sizeof(typename entry_point_type::value_type); - // The IDTranslator holds two tsl::robin_map instances (external->internal and - // internal->external), neither of which exposes its allocated byte count. We - // approximate the storage as the id pair held in each of the two directions. This - // ignores the maps' load-factor slack and control bytes, so it is an estimate of - // the hash-map overhead that is accurate to within a few percent. + // Estimate: the id pair per entry, counted twice (external->internal hash map and + // internal->external vector); ignores hash-map load-factor slack and vector spare + // capacity. metadata_bytes += 2 * translator_.size() * (sizeof(IDTranslator::external_id_type) + sizeof(IDTranslator::internal_id_type)); @@ -790,27 +788,20 @@ class MutableVamanaIndex { auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - auto run = [&](const auto& search) { - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - if constexpr (Filtered) { - run(search); - } else { - run(search); - } + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + search, + *this, + cancel + ); }; if (filter != nullptr) { [&]() __attribute__((noinline)) { body(std::true_type{}); } diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index da4721459..f3911d2ae 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -647,28 +647,21 @@ class VamanaIndex { auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); - auto run = [&](const auto& search) { - // Perform a search over the batch of queries. - extensions::per_thread_batch_search( - data_, - search_buffer, - scratch, - queries, - result, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; auto search = greedy_search_closure( prefetch_parameters, cancel, filter ); - if constexpr (Filtered) { - run(search); - } else { - run(search); - } + // Perform a search over the batch of queries. + extensions::per_thread_batch_search( + data_, + search_buffer, + scratch, + queries, + result, + threads::UnitRange{is}, + search, + *this, + cancel + ); }; // Keep the filtered body out of line so regular search is unaffected. if (filter != nullptr) { diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index e256c42e2..5e1f53a0e 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -614,13 +614,13 @@ class DynamicVamana : public manager::IndexManager { /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. /// ``filter`` must be thread-safe and must not call into the same index. template - bool filtered_search( + void filtered_search( QueryResultView results, data::ConstSimpleDataView queries, const VamanaSearchParameters& search_parameters, const std::function& filter ) { - return impl_->filtered_search( + impl_->filtered_search( results, AnonymousArray<2>(queries), search_parameters, filter ); } diff --git a/include/svs/orchestrators/manager.h b/include/svs/orchestrators/manager.h index b3f4d33d2..3fcbb4ed6 100644 --- a/include/svs/orchestrators/manager.h +++ b/include/svs/orchestrators/manager.h @@ -146,25 +146,11 @@ class ManagerImpl : public ManagerInterface { const search_parameters_type& search_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) override { - // See if we have a specialization for this particular query type. - // If so, invoke that specialization, otherwise throw - lib::match( - QueryTypes{}, - data.type(), - [&](lib::Type SVS_UNUSED(type)) { - const auto view = data::ConstSimpleDataView(data); - svs::index::search_batch_into_with( - implementation_, result, view, search_parameters, cancel - ); - }, - [&](svs::DataType data_type) { - throw ANNEXCEPTION( - "Unsupported datatype! Got: {}. Expected one of: {}.", - data_type, - fmt::join(QueryTypes::data_types(), ", ") - ); - } - ); + dispatch_queries(result, data, [&](const auto& view) { + svs::index::search_batch_into_with( + implementation_, result, view, search_parameters, cancel + ); + }); } // Data Interface @@ -191,6 +177,31 @@ class ManagerImpl : public ManagerInterface { Impl& impl() { return implementation_; } const Impl& impl() const { return implementation_; } + // Invoke `f` with the query view of the matching supported type, otherwise throw. + template + void dispatch_queries( + [[maybe_unused]] QueryResultView result, + const AnonymousArray<2>& data, + F&& f + ) const { + lib::match( + QueryTypes{}, + data.type(), + [&](lib::Type SVS_UNUSED(type)) { + const auto view = data::ConstSimpleDataView(data); + assert(result.n_queries() == view.size()); + f(view); + }, + [&](svs::DataType data_type) { + throw ANNEXCEPTION( + "Unsupported datatype! Got: {}. Expected one of: {}.", + data_type, + fmt::join(QueryTypes::data_types(), ", ") + ); + } + ); + } + private: friend class IndexManager; Impl implementation_; diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index 2637e7f58..e43ba617c 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -111,7 +111,7 @@ class VamanaInterface { virtual svs::index::vamana::VamanaIndexParameters parameters() const = 0; // Filtered search; keep last (ABI). - virtual bool filtered_search( + virtual void filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, @@ -293,29 +293,17 @@ class VamanaImpl : public manager::ManagerImpl { return impl().parameters(); } - bool filtered_search( + void filtered_search( svs::QueryResultView results, AnonymousArray<2> queries, const index::vamana::VamanaSearchParameters& search_parameters, const std::function& filter ) override { - lib::match( - QueryTypes{}, - queries.type(), - [&](lib::Type SVS_UNUSED(type)) { - impl().search( - results, - data::ConstSimpleDataView(queries), - search_parameters, - lib::Returns(lib::Const()), - &filter - ); - }, - [&](svs::DataType data_type) { - throw ANNEXCEPTION("Unsupported query datatype! Got: {}.", data_type); - } - ); - return true; + this->dispatch_queries(results, queries, [&](const auto& view) { + impl().search( + results, view, search_parameters, lib::Returns(lib::Const()), &filter + ); + }); } }; @@ -814,13 +802,13 @@ class Vamana : public manager::IndexManager { /// Rows with fewer passing entries than requested are padded with id ``size_t(-1)``. /// ``filter`` must be thread-safe and must not call into the same index. template - bool filtered_search( + void filtered_search( QueryResultView results, data::ConstSimpleDataView queries, const VamanaSearchParameters& search_parameters, const std::function& filter ) { - return impl_->filtered_search( + impl_->filtered_search( results, AnonymousArray<2>(queries), search_parameters, filter ); } From a68e145a488966f7685668bc94a924e523cfaf2f Mon Sep 17 00:00:00 2001 From: yuejiaointel Date: Thu, 12 Nov 2026 15:52:53 -0800 Subject: [PATCH 47/47] Shrink the filtered search diff --- include/svs/concepts/distance.h | 7 ++ include/svs/concurrent/dynamic_index.h | 131 ++++++++++---------- include/svs/core/translation.h | 3 +- include/svs/index/vamana/dynamic_index.h | 119 +++++++++--------- include/svs/index/vamana/extensions.h | 6 +- include/svs/index/vamana/index.h | 150 ++++++++++++----------- include/svs/orchestrators/manager.h | 9 +- include/svs/orchestrators/vamana.h | 2 +- 8 files changed, 219 insertions(+), 208 deletions(-) diff --git a/include/svs/concepts/distance.h b/include/svs/concepts/distance.h index ec240b88e..7877fab5d 100644 --- a/include/svs/concepts/distance.h +++ b/include/svs/concepts/distance.h @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -215,6 +216,12 @@ template auto comparator(const T& /*x*/) -> compare_t { return typename T::compare{}; } +/// Distance reported for a missing neighbor: it sorts after every real distance. +template float worst_distance(const T& x) { + return comparator(x)(0.0f, 1.0f) ? std::numeric_limits::infinity() + : -std::numeric_limits::infinity(); +} + ///// ///// Traits ///// diff --git a/include/svs/concurrent/dynamic_index.h b/include/svs/concurrent/dynamic_index.h index 48a3ccc8d..7e3b4a0b7 100644 --- a/include/svs/concurrent/dynamic_index.h +++ b/include/svs/concurrent/dynamic_index.h @@ -659,9 +659,7 @@ class MutableVamanaIndex { size_t metadata_bytes = status_.capacity() * sizeof(SlotMetadata); metadata_bytes += entry_point_.capacity() * sizeof(typename entry_point_type::value_type); - // Estimate: the id pair per entry, counted twice (external->internal hash map and - // internal->external vector); ignores hash-map load-factor slack and vector spare - // capacity. + // Estimate: one id pair per entry for each translation direction. metadata_bytes += 2 * translator_.size() * (sizeof(IDTranslator::external_id_type) + sizeof(IDTranslator::internal_id_type)); @@ -975,28 +973,26 @@ class MutableVamanaIndex { buffer.clear(); return; } + auto builder = [&] { + if constexpr (Filtered) { + return internal_search_builder(*filter); + } else { + return internal_search_builder(); + } + }(); // Perform the greedy search using the provided resources. - auto run = [&](auto& builder) { - concurrent::greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{std::span{&seed, 1}}, - builder, - prefetch_parameters, - cancel - ); - }; - if constexpr (Filtered) { - auto builder = internal_search_builder(*filter); - run(builder); - } else { - auto builder = internal_search_builder(); - run(builder); - } + concurrent::greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{std::span{&seed, 1}}, + builder, + prefetch_parameters, + cancel + ); // Take a pass over the search buffer to remove any deleted elements that // might remain. buffer.cleanup(); @@ -1024,8 +1020,6 @@ class MutableVamanaIndex { ); } - // `filter` is called from every search thread without locks; it must be thread-safe - // and must not call into this index. template void search( QueryResultView results, @@ -1041,44 +1035,52 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - auto body = [&](std::bool_constant) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - // Legalize search buffer for this search. - if (buffer.target_capacity() < num_neighbors) { - buffer.change_maxsize(num_neighbors); - } - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - auto search = greedy_search_closure( - prefetch_parameters, cancel, filter - ); - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + // Legalize search buffer for this search. + if (buffer.target_capacity() < num_neighbors) { + buffer.change_maxsize(num_neighbors); + } + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + if (filter != nullptr) { - [&]() __attribute__((noinline)) { body(std::true_type{}); } + [&]() __attribute__((noinline)) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure( + prefetch_parameters, cancel, filter + ), + *this, + cancel + ); + } (); - } else { - body(std::false_type{}); + return; } + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure(prefetch_parameters, cancel), + *this, + cancel + ); } ); } @@ -1093,11 +1095,8 @@ class MutableVamanaIndex { translate_to_external(results.indices()); if (filter != nullptr) { - // A concurrent remap may have changed the answer: only return ids that pass - // now. - const float worst = distance::comparator(distance_)(0.0f, 1.0f) - ? std::numeric_limits::infinity() - : -std::numeric_limits::infinity(); + // A concurrent remap may have changed the answer: re-check the final ids. + const float worst = distance::worst_distance(distance_); for (size_t i = 0; i < results.n_queries(); ++i) { for (size_t j = 0; j < results.n_neighbors(); ++j) { auto id = results.index(i, j); diff --git a/include/svs/core/translation.h b/include/svs/core/translation.h index 259049ae1..1ffa034bc 100644 --- a/include/svs/core/translation.h +++ b/include/svs/core/translation.h @@ -538,8 +538,7 @@ class IDTranslator { } tsl::robin_map external_to_internal_{}; - // TODO: moves to #447 (same vector-based internal-to-external table); not in this PR. - // Segments never move when it grows, so `load_external` needs no lock. + // Vector for lock-free reads; TODO: moves to #447 (same table), not in this PR. lib::SegmentedVector internal_to_external_{}; external_id_type load_external(internal_id_type i) const { diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index ac7131b71..072ecb3c3 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -537,9 +537,7 @@ class MutableVamanaIndex { size_t metadata_bytes = status_.capacity() * sizeof(SlotMetadata); metadata_bytes += entry_point_.capacity() * sizeof(typename entry_point_type::value_type); - // Estimate: the id pair per entry, counted twice (external->internal hash map and - // internal->external vector); ignores hash-map load-factor slack and vector spare - // capacity. + // Estimate: one id pair per entry for each translation direction. metadata_bytes += 2 * translator_.size() * (sizeof(IDTranslator::external_id_type) + sizeof(IDTranslator::internal_id_type)); @@ -715,28 +713,26 @@ class MutableVamanaIndex { prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { static_cast(filter); // Unused in regular search. + auto builder = [&] { + if constexpr (Filtered) { + return internal_search_builder(*filter); + } else { + return internal_search_builder(); + } + }(); // Perform the greedy search using the provided resources. - auto run = [&](auto& builder) { - greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - builder, - prefetch_parameters, - cancel - ); - }; - if constexpr (Filtered) { - auto builder = internal_search_builder(*filter); - run(builder); - } else { - auto builder = internal_search_builder(); - run(builder); - } + greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + builder, + prefetch_parameters, + cancel + ); // Take a pass over the search buffer to remove any deleted elements that // might remain. buffer.cleanup(); @@ -772,43 +768,50 @@ class MutableVamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - auto body = [&](std::bool_constant) { - size_t num_neighbors = results.n_neighbors(); - auto buffer = search_buffer_type{ - // Legalize search buffer for this search. - sp.buffer_config_.get_total_capacity() < num_neighbors - ? SearchBufferConfig{num_neighbors} - : sp.buffer_config_, - distance::comparator(distance_), - sp.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - sp.prefetch_lookahead_, sp.prefetch_step_}; - - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - auto search = greedy_search_closure( - prefetch_parameters, cancel, filter - ); - extensions::per_thread_batch_search( - data_, - buffer, - scratch, - queries, - results, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; + size_t num_neighbors = results.n_neighbors(); + auto buffer = search_buffer_type{ + // Legalize search buffer for this search. + sp.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : sp.buffer_config_, + distance::comparator(distance_), + sp.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + sp.prefetch_lookahead_, sp.prefetch_step_}; + + auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); + if (filter != nullptr) { - [&]() __attribute__((noinline)) { body(std::true_type{}); } + [&]() __attribute__((noinline)) { + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure( + prefetch_parameters, cancel, filter + ), + *this, + cancel + ); + } (); - } else { - body(std::false_type{}); + return; } + extensions::per_thread_batch_search( + data_, + buffer, + scratch, + queries, + results, + threads::UnitRange{is}, + greedy_search_closure(prefetch_parameters, cancel), + *this, + cancel + ); } ); diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index b13e43f21..a1b87ccc2 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -27,8 +27,6 @@ #include "svs/lib/preprocessor.h" #include "svs/lib/threads.h" -#include - namespace svs::index::vamana::extensions { ///// @@ -594,9 +592,7 @@ void svs_invoke( // Fewer than k results (e.g. a selective filter): pad with invalid ids. for (size_t j = num_results; j < num_neighbors; ++j) { result.index(i, j) = static_cast(-1); - result.distance(i, j) = distance::comparator(distance)(0.0f, 1.0f) - ? std::numeric_limits::infinity() - : -std::numeric_limits::infinity(); + result.distance(i, j) = distance::worst_distance(distance); } } } diff --git a/include/svs/index/vamana/index.h b/include/svs/index/vamana/index.h index f3911d2ae..bce8d8b81 100644 --- a/include/svs/index/vamana/index.h +++ b/include/svs/index/vamana/index.h @@ -513,29 +513,28 @@ class VamanaIndex { prefetch_parameters, filter](const auto& query, auto& accessor, auto& distance, auto& buffer) { static_cast(filter); // Unused in regular search. - auto run = [&](auto& builder) { - greedy_search( - graph_, - data_, - accessor, - query, - distance, - buffer, - vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, - builder, - prefetch_parameters, - cancel - ); - }; + auto builder = [&] { + if constexpr (Filtered) { + return FilteredNeighborBuilder>{*filter}; + } else { + return internal_search_builder(); + } + }(); + greedy_search( + graph_, + data_, + accessor, + query, + distance, + buffer, + vamana::EntryPointInitializer{lib::as_const_span(entry_point_)}, + builder, + prefetch_parameters, + cancel + ); if constexpr (Filtered) { - auto builder = - FilteredNeighborBuilder>{*filter}; - run(builder); // Remove the invalid entries that remain in the buffer. buffer.cleanup(); - } else { - auto builder = internal_search_builder(); - run(builder); } }; } @@ -618,58 +617,71 @@ class VamanaIndex { threadpool_, threads::StaticPartition{queries.size()}, [&](const auto is, uint64_t SVS_UNUSED(tid)) { - auto body = [&](std::bool_constant) { - // Filtered search needs a buffer that can hold invalid entries. - using buffer_type = std::conditional_t< - Filtered, - MutableBuffer>, - search_buffer_type>; - - // The number of neighbors to store in the result. - size_t num_neighbors = result.n_neighbors(); - - // Allocate scratchspace according to the provided search parameters. - auto search_buffer = buffer_type{ - // Increase the search window size if the defaults are not - // suitable for the requested number of neighbors. - search_parameters.buffer_config_.get_total_capacity() < - num_neighbors - ? SearchBufferConfig{num_neighbors} - : search_parameters.buffer_config_, - distance::comparator(distance_), - search_parameters.search_buffer_visited_set_}; - - auto prefetch_parameters = GreedySearchPrefetchParameters{ - search_parameters.prefetch_lookahead_, - search_parameters.prefetch_step_}; - - // Pre-allocate scratch space needed by the dataset implementation. - auto scratch = - extensions::per_thread_batch_search_setup(data_, distance_); - - auto search = greedy_search_closure( - prefetch_parameters, cancel, filter - ); - // Perform a search over the batch of queries. - extensions::per_thread_batch_search( - data_, - search_buffer, - scratch, - queries, - result, - threads::UnitRange{is}, - search, - *this, - cancel - ); - }; - // Keep the filtered body out of line so regular search is unaffected. if (filter != nullptr) { - [&]() __attribute__((noinline)) { body(std::true_type{}); } + [&]() __attribute__((noinline)) { + // The filtered search buffer must hold invalid entries. + auto search_buffer = MutableBuffer>{ + search_parameters.buffer_config_.get_total_capacity() < + result.n_neighbors() + ? SearchBufferConfig{result.n_neighbors()} + : search_parameters.buffer_config_, + distance::comparator(distance_), + search_parameters.search_buffer_visited_set_}; + auto prefetch_parameters = GreedySearchPrefetchParameters{ + search_parameters.prefetch_lookahead_, + search_parameters.prefetch_step_}; + auto scratch = + extensions::per_thread_batch_search_setup(data_, distance_); + extensions::per_thread_batch_search( + data_, + search_buffer, + scratch, + queries, + result, + threads::UnitRange{is}, + greedy_search_closure( + prefetch_parameters, cancel, filter + ), + *this, + cancel + ); + } (); - } else { - body(std::false_type{}); + return; } + + // The number of neighbors to store in the result. + size_t num_neighbors = result.n_neighbors(); + + // Allocate scratchspace according to the provided search parameters. + auto search_buffer = search_buffer_type{ + // Increase the search window size if the defaults are not suitable for + // the requested number of neighbors. + search_parameters.buffer_config_.get_total_capacity() < num_neighbors + ? SearchBufferConfig{num_neighbors} + : search_parameters.buffer_config_, + distance::comparator(distance_), + search_parameters.search_buffer_visited_set_}; + + auto prefetch_parameters = GreedySearchPrefetchParameters{ + search_parameters.prefetch_lookahead_, + search_parameters.prefetch_step_}; + + // Pre-allocate scratch space needed by the dataset implementation. + auto scratch = extensions::per_thread_batch_search_setup(data_, distance_); + + // Perform a search over the batch of queries. + extensions::per_thread_batch_search( + data_, + search_buffer, + scratch, + queries, + result, + threads::UnitRange{is}, + greedy_search_closure(prefetch_parameters, cancel), + *this, + cancel + ); } ); } diff --git a/include/svs/orchestrators/manager.h b/include/svs/orchestrators/manager.h index 3fcbb4ed6..1625f256d 100644 --- a/include/svs/orchestrators/manager.h +++ b/include/svs/orchestrators/manager.h @@ -146,7 +146,7 @@ class ManagerImpl : public ManagerInterface { const search_parameters_type& search_parameters, const lib::DefaultPredicate& cancel = lib::Returns(lib::Const()) ) override { - dispatch_queries(result, data, [&](const auto& view) { + dispatch_queries(data, [&](const auto& view) { svs::index::search_batch_into_with( implementation_, result, view, search_parameters, cancel ); @@ -179,17 +179,12 @@ class ManagerImpl : public ManagerInterface { // Invoke `f` with the query view of the matching supported type, otherwise throw. template - void dispatch_queries( - [[maybe_unused]] QueryResultView result, - const AnonymousArray<2>& data, - F&& f - ) const { + void dispatch_queries(const AnonymousArray<2>& data, F&& f) const { lib::match( QueryTypes{}, data.type(), [&](lib::Type SVS_UNUSED(type)) { const auto view = data::ConstSimpleDataView(data); - assert(result.n_queries() == view.size()); f(view); }, [&](svs::DataType data_type) { diff --git a/include/svs/orchestrators/vamana.h b/include/svs/orchestrators/vamana.h index e43ba617c..cada48419 100644 --- a/include/svs/orchestrators/vamana.h +++ b/include/svs/orchestrators/vamana.h @@ -299,7 +299,7 @@ class VamanaImpl : public manager::ManagerImpl { const index::vamana::VamanaSearchParameters& search_parameters, const std::function& filter ) override { - this->dispatch_queries(results, queries, [&](const auto& view) { + this->dispatch_queries(queries, [&](const auto& view) { impl().search( results, view, search_parameters, lib::Returns(lib::Const()), &filter );