From c8792e42b86a80a93a584c9bc792e315ee4ed2d6 Mon Sep 17 00:00:00 2001 From: Kristofferjoha Date: Sat, 7 Nov 2026 10:21:07 +0100 Subject: [PATCH] clustering --- README.md | 1 + engine/CMakeLists.txt | 3 + engine/configs/geolife.json | 7 +- .../backend/engine/evaluator/combined.hpp | 5 +- .../backend/engine/evaluator/evaluator.hpp | 5 + .../engine/evaluator/route_clustering.hpp | 34 +++ .../backend/engine/pipeline/config.hpp | 2 + .../backend/engine/query/route_clustering.hpp | 58 ++++ engine/src/evaluator/combined.cpp | 5 +- engine/src/evaluator/evaluator.cpp | 8 +- engine/src/evaluator/route_clustering.cpp | 95 +++++++ engine/src/pipeline/config.cpp | 16 +- engine/src/pipeline/pipeline.cpp | 30 +- engine/src/query/route_clustering.cpp | 124 +++++++++ engine/tests/config_test.cpp | 25 ++ engine/tests/route_clustering_test.cpp | 256 ++++++++++++++++++ 16 files changed, 665 insertions(+), 9 deletions(-) create mode 100644 engine/include/backend/engine/evaluator/route_clustering.hpp create mode 100644 engine/include/backend/engine/query/route_clustering.hpp create mode 100644 engine/src/evaluator/route_clustering.cpp create mode 100644 engine/src/query/route_clustering.cpp create mode 100644 engine/tests/route_clustering_test.cpp diff --git a/README.md b/README.md index c11673a..ea1058f 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ and it then falls back to its default. [engine/configs/geolife.json](engine/conf | `range` | Box size (m), time window (s) and sampling limits for range queries | | `where_at` | Probes per trajectory and max time gap (s) between points to probe | | `where_at_scoring` | Hit radius and tolerance range (m) for scoring where-at queries | +| `clustering` | Route clustering metric: `enabled`, neighbour threshold `eps_m` (m) and `min_size` per cluster. Reuses the self join when `join.mode` is `Self` with the same eps | | `allocation_alphas` | Allocation exponents to test: budgets follow size^α (`1` = size-proportional, `0` = equal). Exports for α ≠ 1 get an `_a<α>` suffix | ## Tests diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index a492dd9..5253712 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -37,6 +37,8 @@ add_library(engine src/query/join.cpp src/evaluator/join.cpp src/evaluator/combined.cpp + src/query/route_clustering.cpp + src/evaluator/route_clustering.cpp ) target_include_directories(engine PUBLIC include) @@ -69,6 +71,7 @@ add_executable(engine_tests tests/config_test.cpp tests/join_test.cpp tests/combined_test.cpp + tests/route_clustering_test.cpp ) target_link_libraries(engine_tests PRIVATE engine GTest::gtest_main Arrow::arrow_shared Parquet::parquet_shared) gtest_discover_tests(engine_tests) diff --git a/engine/configs/geolife.json b/engine/configs/geolife.json index 11fb1db..bf413f9 100644 --- a/engine/configs/geolife.json +++ b/engine/configs/geolife.json @@ -1,5 +1,5 @@ { - "data_path": "data/geolife.parquet", + "data_path": "data/geolife_gap60.parquet", "export_results": true, "export_dir": "data/output/geolife", "seed": 42, @@ -34,5 +34,10 @@ "join": { "mode": "Self", "eps_m": 500.0 + }, + "clustering": { + "enabled": false, + "eps_m": 150.0, + "min_size": 5 } } diff --git a/engine/include/backend/engine/evaluator/combined.hpp b/engine/include/backend/engine/evaluator/combined.hpp index 3d4256d..0472d46 100644 --- a/engine/include/backend/engine/evaluator/combined.hpp +++ b/engine/include/backend/engine/evaluator/combined.hpp @@ -7,7 +7,8 @@ namespace evaluator { // 0 = the floor (endpoints only), 1 = perfect, without clipping. Combined averages -// one primary score per type: recall@k, range F1, where-at within-radius hit rate, join F1. +// one primary score per type: recall@k, range F1, where-at within-radius hit rate, join F1 +// and route clustering. // Missing primaries are skipped, never replaced with a diagnostic score. // The radius is configured by hit_radius_m (100 m for the research protocol). // Average-hit remains a diagnostic and does not contribute to the combined score. @@ -21,11 +22,13 @@ struct NormalizedScores { double where_at_within_radius = missing; double where_at_average_hit = missing; double join_f1 = missing; + double clustering_score = missing; double knn = missing; double range = missing; double where_at = missing; double join = missing; + double clustering = missing; double combined = missing; }; diff --git a/engine/include/backend/engine/evaluator/evaluator.hpp b/engine/include/backend/engine/evaluator/evaluator.hpp index dbb2d70..f2f38f0 100644 --- a/engine/include/backend/engine/evaluator/evaluator.hpp +++ b/engine/include/backend/engine/evaluator/evaluator.hpp @@ -3,10 +3,12 @@ #include "backend/engine/core/trajectory_db.hpp" #include "backend/engine/evaluator/join.hpp" #include "backend/engine/evaluator/knn.hpp" +#include "backend/engine/evaluator/route_clustering.hpp" #include "backend/engine/evaluator/where_at.hpp" #include "backend/engine/query/join.hpp" #include "backend/engine/query/knn.hpp" #include "backend/engine/query/range_query.hpp" +#include "backend/engine/query/route_clustering.hpp" #include #include @@ -19,6 +21,7 @@ struct Workloads { std::span where_at = {}; std::span range = {}; const query::JoinWorkload *join = nullptr; // a pointer, as it also carries eps and the queries + const query::RouteClusteringWorkload *clustering = nullptr; }; // Kendall's tau is in [-1, 1] and the others in [0, 1], or NaN if it could not be computed @@ -28,6 +31,7 @@ struct Scoretable { WhereAtScores where_at; JoinScores join; double range_f1 = std::numeric_limits::quiet_NaN(); + RouteClusteringScores clustering; }; struct EvaluationResult { @@ -49,6 +53,7 @@ class Evaluator { double get_range_f1() const; WhereAtScores get_where_at_scores() const; JoinScores get_join_scores() const; + RouteClusteringScores get_clustering_scores() const; private: const core::TrajectoryDb &m_simplified; diff --git a/engine/include/backend/engine/evaluator/route_clustering.hpp b/engine/include/backend/engine/evaluator/route_clustering.hpp new file mode 100644 index 0000000..940f238 --- /dev/null +++ b/engine/include/backend/engine/evaluator/route_clustering.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/query/route_clustering.hpp" + +#include +#include + +namespace evaluator { + +// Reruns the route clustering on the compressed database and compares it with the reference. +// score is the symmetric macro best-match F1: each cluster's best overlap F1 with any cluster on +// the other side, averaged per side, then over both sides. NaN without reference clusters, 0 if +// nothing is predicted. +struct RouteClusteringScores { + static constexpr double missing = std::numeric_limits::quiet_NaN(); + + double score = missing; + std::size_t reference_clusters = 0; + std::size_t predicted_clusters = 0; + std::size_t reference_covered = 0; + std::size_t predicted_covered = 0; +}; + +query::RouteClusters compressed_route_clusters(const core::TrajectoryDb &compressed, + const query::RouteClusteringWorkload &workload); + +RouteClusteringScores compare_route_clusters(const query::RouteClusters &reference, + const query::RouteClusters &predicted); + +RouteClusteringScores evaluate_route_clustering(const core::TrajectoryDb &compressed, + const query::RouteClusteringWorkload &workload); + +} // namespace evaluator diff --git a/engine/include/backend/engine/pipeline/config.hpp b/engine/include/backend/engine/pipeline/config.hpp index 965903f..74f47db 100644 --- a/engine/include/backend/engine/pipeline/config.hpp +++ b/engine/include/backend/engine/pipeline/config.hpp @@ -4,6 +4,7 @@ #include "backend/engine/query/join.hpp" #include "backend/engine/query/knn.hpp" #include "backend/engine/query/range_query.hpp" +#include "backend/engine/query/route_clustering.hpp" #include "backend/engine/query/where_at_query.hpp" #include "backend/engine/selection/selector.hpp" #include "backend/engine/split/split.hpp" @@ -29,6 +30,7 @@ struct Config { query::WhereAtConfig where_at; evaluator::WhereAtScoringConfig where_at_scoring; query::JoinConfig join; + query::RouteClusteringConfig clustering; }; // Reads a JSON config. Only data_path is required, a missing key keeps its default. diff --git a/engine/include/backend/engine/query/route_clustering.hpp b/engine/include/backend/engine/query/route_clustering.hpp new file mode 100644 index 0000000..2161372 --- /dev/null +++ b/engine/include/backend/engine/query/route_clustering.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/query/join.hpp" + +#include +#include +#include +#include +#include + +namespace query { + +struct RouteClusteringConfig { + bool enabled = false; + double eps_m = 500.0; // neighbour threshold, same test as the self join + std::size_t min_size = 5; // a cluster needs at least this many trajectories, medoid included +}; + +using TrajectoryPair = std::pair; + +// Greedy medoid grouping. Repeatedly takes the unassigned trajectory with the most unassigned +// neighbours and groups it with them if that gives at least min_size trajectories. Every member +// neighbours its medoid, so clusters cannot chain. Ties go to the earliest start time, then the +// smallest start x, then the smallest trajectory ID, which makes the result order independent. +struct RouteClusters { + std::vector label; // cluster per trajectory, -1 = unclustered + std::vector> members; // members[c][0] is the medoid +}; + +RouteClusters cluster_routes(const core::TrajectoryDb &trajectories, + std::span neighbours, std::size_t min_size); + +// ID and endpoints of an original trajectory. The candidate pairs were pruned by endpoint +// distance, so a database is only comparable if every one of these matches. +struct TrajectoryEndpoints { + std::int32_t trajectory_id = 0; + double first_x = 0.0, first_y = 0.0, last_x = 0.0, last_y = 0.0; + bool operator==(const TrajectoryEndpoints &) const = default; +}; + +TrajectoryEndpoints endpoints_of(const core::TrajectoryView &trajectory); + +struct RouteClusteringWorkload { + std::vector candidates; // endpoint-pruned pairs, i < j + RouteClusters reference; + std::vector originals; + double eps_m = 0.0; + std::size_t min_size = 0; +}; + +// Clusters the original database. A self join workload over the same trajectories and eps is +// reused instead of recomputing the Fréchet tests on the originals. +RouteClusteringWorkload generate_route_clustering(const core::TrajectoryDb &original, + const RouteClusteringConfig &config = {}, + const JoinWorkload *self_join = nullptr); + +} // namespace query diff --git a/engine/src/evaluator/combined.cpp b/engine/src/evaluator/combined.cpp index 2be9b7b..19a8f27 100644 --- a/engine/src/evaluator/combined.cpp +++ b/engine/src/evaluator/combined.cpp @@ -41,13 +41,16 @@ NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &fl result.where_at_average_hit = normalize(scores.where_at.average_hit, floor.where_at.average_hit); result.join_f1 = normalize(scores.join.f1, floor.join.f1); + result.clustering_score = normalize(scores.clustering.score, floor.clustering.score); result.knn = result.knn_recall_at_k; result.range = result.range_f1; result.where_at = result.where_at_within_radius; result.join = result.join_f1; + result.clustering = result.clustering_score; - result.combined = mean_ignoring_nan({result.knn, result.range, result.where_at, result.join}); + result.combined = mean_ignoring_nan( + {result.knn, result.range, result.where_at, result.join, result.clustering}); return result; } diff --git a/engine/src/evaluator/evaluator.cpp b/engine/src/evaluator/evaluator.cpp index 9faf8a7..5528124 100644 --- a/engine/src/evaluator/evaluator.cpp +++ b/engine/src/evaluator/evaluator.cpp @@ -27,7 +27,8 @@ EvaluationResult Evaluator::evaluate() const { .range_recall = range.recall, .where_at = get_where_at_scores(), .join = get_join_scores(), - .range_f1 = range.f1}, + .range_f1 = range.f1, + .clustering = get_clustering_scores()}, .knn = std::move(knn.answers), .range = std::move(range.answers)}; } @@ -52,4 +53,9 @@ JoinScores Evaluator::get_join_scores() const { return m_workloads.join ? evaluate_join(m_simplified, *m_workloads.join) : JoinScores{}; } +RouteClusteringScores Evaluator::get_clustering_scores() const { + return m_workloads.clustering ? evaluate_route_clustering(m_simplified, *m_workloads.clustering) + : RouteClusteringScores{}; +} + } // namespace evaluator diff --git a/engine/src/evaluator/route_clustering.cpp b/engine/src/evaluator/route_clustering.cpp new file mode 100644 index 0000000..4a30cd9 --- /dev/null +++ b/engine/src/evaluator/route_clustering.cpp @@ -0,0 +1,95 @@ +#include "backend/engine/evaluator/route_clustering.hpp" +#include "backend/engine/core/parallel.hpp" +#include "backend/engine/query/frechet_helper.hpp" + +#include +#include +#include +#include +#include + +namespace evaluator { + +namespace { + +// Mean over a's clusters of the best overlap F1 with any of b's clusters +double best_match(const query::RouteClusters &a, const query::RouteClusters &b) { + double sum = 0.0; + for (const auto &members : a.members) { + std::unordered_map overlap; + for (const auto i : members) { + if (b.label[i] >= 0) + ++overlap[b.label[i]]; + } + double best = 0.0; + for (const auto &[cluster, shared] : overlap) { + const auto size = members.size() + b.members[cluster].size(); + best = std::max(best, 2.0 * static_cast(shared) / static_cast(size)); + } + sum += best; + } + return sum / static_cast(a.members.size()); +} + +std::size_t covered(const query::RouteClusters &clusters) { + std::size_t total = 0; + for (const auto &members : clusters.members) + total += members.size(); + return total; +} + +} // namespace + +query::RouteClusters compressed_route_clusters(const core::TrajectoryDb &compressed, + const query::RouteClusteringWorkload &workload) { + if (compressed.size() != workload.originals.size()) + throw std::invalid_argument("route clustering: database size does not match reference"); + // Simplification keeps endpoints, so the original candidates cover every possible match + for (std::size_t i = 0; i < compressed.size(); ++i) { + const auto trajectory = compressed.trajectory(i); + if (trajectory.size() == 0 || query::endpoints_of(trajectory) != workload.originals[i]) + throw std::invalid_argument( + "route clustering: IDs/order or endpoints do not match reference"); + } + + std::vector matched(workload.candidates.size(), 0); + core::parallel_for(workload.candidates.size(), [&](std::size_t k) { + const auto [i, j] = workload.candidates[k]; + matched[k] = query::frechet_within(compressed.trajectory(i), compressed.trajectory(j), + workload.eps_m); + }); + std::vector neighbours; + for (std::size_t k = 0; k < matched.size(); ++k) { + if (matched[k]) + neighbours.push_back(workload.candidates[k]); + } + return query::cluster_routes(compressed, neighbours, workload.min_size); +} + +RouteClusteringScores compare_route_clusters(const query::RouteClusters &reference, + const query::RouteClusters &predicted) { + if (reference.label.size() != predicted.label.size()) + throw std::invalid_argument("route clustering: label sizes differ"); + + RouteClusteringScores scores; + scores.reference_clusters = reference.members.size(); + scores.predicted_clusters = predicted.members.size(); + scores.reference_covered = covered(reference); + scores.predicted_covered = covered(predicted); + if (reference.members.empty()) + return scores; + + scores.score = + predicted.members.empty() + ? 0.0 + : 0.5 * (best_match(reference, predicted) + best_match(predicted, reference)); + return scores; +} + +RouteClusteringScores evaluate_route_clustering(const core::TrajectoryDb &compressed, + const query::RouteClusteringWorkload &workload) { + return compare_route_clusters(workload.reference, + compressed_route_clusters(compressed, workload)); +} + +} // namespace evaluator diff --git a/engine/src/pipeline/config.cpp b/engine/src/pipeline/config.cpp index bc48c14..0785f64 100644 --- a/engine/src/pipeline/config.cpp +++ b/engine/src/pipeline/config.cpp @@ -92,11 +92,24 @@ void parse_join_config(Config &config, const json &join) { config.join.eps_m = join.value("eps_m", config.join.eps_m); } +void parse_clustering_config(Config &config, const json &clustering) { + validate_keys(clustering, {"enabled", "eps_m", "min_size"}, "clustering"); + config.clustering.enabled = clustering.value("enabled", config.clustering.enabled); + config.clustering.eps_m = clustering.value("eps_m", config.clustering.eps_m); + if (clustering.contains("min_size")) { + // Checked before conversion, since a negative value would wrap around in size_t + const auto &min_size = clustering["min_size"]; + if (!min_size.is_number_unsigned() || min_size.get() < 2) + throw std::invalid_argument("config: clustering.min_size must be an integer >= 2"); + config.clustering.min_size = min_size.get(); + } +} + Config parse_config(const json &data) { validate_keys(data, {"data_path", "export_results", "export_dir", "seed", "budget_shares", "allocation_alphas", "algorithms", "split", "knn", "range", "where_at", - "where_at_scoring", "join"}, + "where_at_scoring", "join", "clustering"}, "root"); Config config; @@ -130,6 +143,7 @@ Config parse_config(const json &data) { parse_where_at_config(config, data.value("where_at", json::object())); parse_where_at_scoring_config(config, data.value("where_at_scoring", json::object())); parse_join_config(config, data.value("join", json::object())); + parse_clustering_config(config, data.value("clustering", json::object())); return config; } diff --git a/engine/src/pipeline/pipeline.cpp b/engine/src/pipeline/pipeline.cpp index b566f9d..3ffe51c 100644 --- a/engine/src/pipeline/pipeline.cpp +++ b/engine/src/pipeline/pipeline.cpp @@ -9,6 +9,7 @@ #include "backend/engine/query/join.hpp" #include "backend/engine/query/knn.hpp" #include "backend/engine/query/range_query.hpp" +#include "backend/engine/query/route_clustering.hpp" #include "backend/engine/query/where_at_query.hpp" #include "backend/engine/selection/selector.hpp" #include "backend/engine/split/split.hpp" @@ -59,12 +60,18 @@ void print_scores(const evaluator::Scoretable &scores) { std::cout << " range F1: " << scores.range_f1 << ", recall: " << scores.range_recall << '\n'; std::cout << " join F1: " << scores.join.f1 << ", precision: " << scores.join.precision << ", recall: " << scores.join.recall << '\n'; + if (!std::isnan(scores.clustering.score) || scores.clustering.reference_clusters != 0) + std::cout << " route clustering: " << scores.clustering.score << " (" + << scores.clustering.predicted_clusters << " clusters, " + << scores.clustering.predicted_covered << " trajectories)\n"; } void print_normalized(const evaluator::NormalizedScores &scores) { std::cout << " normalized: knn=" << scores.knn << ", range=" << scores.range - << ", where-at=" << scores.where_at << ", join=" << scores.join - << ", combined=" << scores.combined << '\n'; + << ", where-at=" << scores.where_at << ", join=" << scores.join; + if (!std::isnan(scores.clustering)) + std::cout << ", clustering=" << scores.clustering; + std::cout << ", combined=" << scores.combined << '\n'; } core::Simplification endpoints_only(const core::TrajectoryDb &trajectories) { @@ -105,13 +112,28 @@ void run(const Config &config) { : " queries, eps=") << join.eps_m << " m, " << join_pairs << " matching pairs\n"; + query::RouteClusteringWorkload clustering; + if (config.clustering.enabled) { + clustering = query::generate_route_clustering(database, config.clustering, &join); + std::size_t covered = 0; + for (const auto &members : clustering.reference.members) + covered += members.size(); + std::cout << "route clustering: " << clustering.reference.members.size() + << " clusters covering " << covered << " trajectories, eps=" << clustering.eps_m + << " m, min size " << clustering.min_size << '\n'; + } + if (config.export_results) { io::export_workloads(config.export_dir, database, range.queries, knn_workload); std::cout << "workloads written to " << config.export_dir.string() << '\n'; } - const evaluator::Workloads workloads{ - .knn = &knn_workload, .where_at = where_at.queries, .range = range.queries, .join = &join}; + const evaluator::Workloads workloads{.knn = &knn_workload, + .where_at = where_at.queries, + .range = range.queries, + .join = &join, + .clustering = + config.clustering.enabled ? &clustering : nullptr}; const auto floor_db = core::materialize(database, endpoints_only(database)); std::cout << "floor, endpoints only: "; diff --git a/engine/src/query/route_clustering.cpp b/engine/src/query/route_clustering.cpp new file mode 100644 index 0000000..b173b8f --- /dev/null +++ b/engine/src/query/route_clustering.cpp @@ -0,0 +1,124 @@ +#include "backend/engine/query/route_clustering.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace query { + +TrajectoryEndpoints endpoints_of(const core::TrajectoryView &trajectory) { + if (trajectory.size() == 0) + throw std::invalid_argument("route clustering: empty trajectory"); + return {trajectory.trajectory_id, trajectory.x.front(), trajectory.y.front(), + trajectory.x.back(), trajectory.y.back()}; +} + +RouteClusters cluster_routes(const core::TrajectoryDb &trajectories, + std::span neighbours, std::size_t min_size) { + if (min_size < 2) + throw std::invalid_argument("route clustering: min_size must be at least 2"); + const std::size_t n = trajectories.size(); + std::vector> adjacency(n); + for (const auto &[i, j] : neighbours) { + if (i >= n || j >= n || i == j) + throw std::invalid_argument("route clustering: invalid neighbour pair"); + adjacency[i].push_back(j); + adjacency[j].push_back(i); + } + // A repeated pair must not count twice towards the degree + for (auto &list : adjacency) { + std::ranges::sort(list); + list.erase(std::ranges::unique(list).begin(), list.end()); + } + + // Max-heap on (degree, then tie-breaks). Degrees only fall, so stale entries are re-pushed. + using Key = std::tuple; + const auto key = [&](std::size_t i, std::size_t degree) { + const auto trajectory = trajectories.trajectory(i); + if (trajectory.size() == 0) + throw std::invalid_argument("route clustering: empty trajectory"); + return Key{degree, -trajectory.t.front(), -trajectory.x.front(), + -static_cast(trajectory.trajectory_id), i}; + }; + std::priority_queue heap; + for (std::size_t i = 0; i < n; ++i) + heap.push(key(i, adjacency[i].size())); + + RouteClusters clusters; + clusters.label.assign(n, -1); + while (!heap.empty()) { + const auto [degree, neg_t, neg_x, neg_id, i] = heap.top(); + heap.pop(); + if (clusters.label[i] >= 0) + continue; + std::size_t free = 0; + for (const auto j : adjacency[i]) + free += clusters.label[j] < 0; + if (free < degree) { + heap.push(key(i, free)); + continue; + } + if (free + 1 < min_size) + break; + const auto id = static_cast(clusters.members.size()); + auto &members = clusters.members.emplace_back(); + members.push_back(i); + clusters.label[i] = id; + for (const auto j : adjacency[i]) { + if (clusters.label[j] < 0) { + clusters.label[j] = id; + members.push_back(j); + } + } + } + return clusters; +} + +RouteClusteringWorkload generate_route_clustering(const core::TrajectoryDb &original, + const RouteClusteringConfig &config, + const JoinWorkload *self_join) { + if (!std::isfinite(config.eps_m) || config.eps_m <= 0.0) + throw std::invalid_argument("route clustering: eps_m must be finite and positive"); + + RouteClusteringWorkload workload; + workload.eps_m = config.eps_m; + workload.min_size = config.min_size; + workload.originals.reserve(original.size()); + for (std::size_t i = 0; i < original.size(); ++i) + workload.originals.push_back(endpoints_of(original.trajectory(i))); + + // A self join stores its database's IDs and endpoints, which must be these trajectories' + const auto same_database = [&] { + if (self_join->queries.size() != original.size() || + self_join->entries.size() != original.size()) + return false; + for (std::size_t i = 0; i < original.size(); ++i) { + const auto stored = self_join->queries.trajectory(i); + if (stored.size() == 0 || endpoints_of(stored) != workload.originals[i]) + return false; + } + return true; + }; + std::vector reference; + const bool reuse = self_join && self_join->mode == JoinMode::Self && + self_join->eps_m == config.eps_m && same_database(); + if (reuse) { + for (const auto &entry : self_join->entries) { + for (const auto j : entry.candidates) + workload.candidates.emplace_back(entry.query_index, j); + for (const auto j : entry.reference) + reference.emplace_back(entry.query_index, j); + } + } else { + const auto join = generate_self_join(original, config.eps_m); + return generate_route_clustering(original, config, &join); + } + workload.reference = cluster_routes(original, reference, config.min_size); + return workload; +} + +} // namespace query diff --git a/engine/tests/config_test.cpp b/engine/tests/config_test.cpp index b6cc8cb..c8f31c3 100644 --- a/engine/tests/config_test.cpp +++ b/engine/tests/config_test.cpp @@ -182,3 +182,28 @@ TEST(Config, RejectsEmptyAllocationAlphas) { "no_alphas.json", R"({"data_path": "d", "allocation_alphas": []})")), std::invalid_argument); } + +TEST(Config, ReadsClusteringKeys) { + const auto config = pipeline::load_config(write_config("clustering.json", R"({ + "data_path": "d", "clustering": {"enabled": true, "eps_m": 250.0, "min_size": 4} + })")); + EXPECT_TRUE(config.clustering.enabled); + EXPECT_DOUBLE_EQ(config.clustering.eps_m, 250.0); + EXPECT_EQ(config.clustering.min_size, 4u); +} + +TEST(Config, RejectsInvalidClustering) { + EXPECT_THROW( + pipeline::load_config(write_config("clustering_small.json", + R"({"data_path": "d", "clustering": {"min_size": 1}})")), + std::invalid_argument); + EXPECT_THROW(pipeline::load_config(write_config( + "clustering_key.json", R"({"data_path": "d", "clustering": {"minsize": 5}})")), + std::invalid_argument); + for (const char *value : {"-1", "2.5", "\"5\""}) + EXPECT_THROW( + pipeline::load_config(write_config( + "clustering_bad.json", + std::string(R"({"data_path": "d", "clustering": {"min_size": )") + value + "}}")), + std::invalid_argument); +} diff --git a/engine/tests/route_clustering_test.cpp b/engine/tests/route_clustering_test.cpp new file mode 100644 index 0000000..2616e86 --- /dev/null +++ b/engine/tests/route_clustering_test.cpp @@ -0,0 +1,256 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "backend/engine/core/simplification.hpp" +#include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/evaluator/combined.hpp" +#include "backend/engine/evaluator/route_clustering.hpp" +#include "backend/engine/query/join.hpp" +#include "backend/engine/query/route_clustering.hpp" + +namespace { + +using Track = std::vector>; // (lat, lon) per point + +core::TrajectoryDb make_db(const std::vector &tracks, + const std::vector &ids = {}) { + std::vector offsets{0}; + std::vector trajectory_ids; + std::vector t; + std::vector lat; + std::vector lon; + for (std::size_t i = 0; i < tracks.size(); ++i) { + trajectory_ids.push_back(ids.empty() ? static_cast(i) : ids[i]); + for (std::size_t k = 0; k < tracks[i].size(); ++k) { + t.push_back(static_cast(k)); + lat.push_back(tracks[i][k].first); + lon.push_back(tracks[i][k].second); + } + offsets.push_back(t.size()); + } + return {std::move(offsets), std::move(trajectory_ids), std::move(t), std::move(lat), + std::move(lon)}; +} + +// One short track per trajectory, start points spread along longitude so tie-breaks are distinct +core::TrajectoryDb spread_db(std::size_t n) { + std::vector tracks; + for (std::size_t i = 0; i < n; ++i) { + const double lon = 116.30 + 0.01 * static_cast(i); + tracks.push_back({{39.90, lon}, {39.91, lon}}); + } + return make_db(tracks); +} + +std::set> id_sets(const core::TrajectoryDb &db, + const query::RouteClusters &clusters) { + std::set> result; + for (const auto &members : clusters.members) { + std::set ids; + for (const auto i : members) + ids.insert(db.trajectory(i).trajectory_id); + result.insert(ids); + } + return result; +} + +// Trajectories on a few shared routes with small offsets, so routes form clusters +std::vector route_tracks(std::mt19937_64 &rng, std::size_t routes, std::size_t per_route) { + std::normal_distribution jitter(0.0, 0.0003); + std::vector tracks; + for (std::size_t r = 0; r < routes; ++r) { + for (std::size_t m = 0; m < per_route; ++m) { + Track track; + for (std::size_t k = 0; k < 20; ++k) { + const double s = static_cast(k) / 19.0; + track.emplace_back(39.90 + 0.05 * static_cast(r) + 0.02 * s + jitter(rng), + 116.30 + 0.03 * s + 0.01 * std::sin(6.0 * s) + jitter(rng)); + } + tracks.push_back(track); + } + } + return tracks; +} + +} // namespace + +TEST(RouteClustering, StarFormsOneClusterAndLeavesNoiseUnclustered) { + const auto db = spread_db(7); + const std::vector pairs{{0, 1}, {0, 2}, {0, 3}, {0, 4}}; + const auto clusters = query::cluster_routes(db, pairs, 5); + ASSERT_EQ(clusters.members.size(), 1u); + EXPECT_EQ(clusters.members[0].front(), 0u); + EXPECT_EQ(std::set(clusters.members[0].begin(), clusters.members[0].end()), + (std::set{0, 1, 2, 3, 4})); + EXPECT_EQ(clusters.label[5], -1); + EXPECT_EQ(clusters.label[6], -1); + EXPECT_TRUE(query::cluster_routes(db, pairs, 6).members.empty()); +} + +TEST(RouteClustering, MembersMustNeighbourTheMedoidSoPathsDoNotChain) { + const auto db = spread_db(5); + const std::vector path{{0, 1}, {1, 2}, {2, 3}, {3, 4}}; + const auto clusters = query::cluster_routes(db, path, 3); + for (const auto &members : clusters.members) { + EXPECT_LE(members.size(), 3u); + const auto medoid = members.front(); + for (std::size_t k = 1; k < members.size(); ++k) { + const auto a = std::min(medoid, members[k]); + const auto b = std::max(medoid, members[k]); + EXPECT_NE(std::find(path.begin(), path.end(), query::TrajectoryPair{a, b}), path.end()); + } + } + // Ties on degree go to the smallest start x, which is trajectory 1 + ASSERT_EQ(clusters.members.size(), 1u); + EXPECT_EQ(clusters.members[0].front(), 1u); +} + +TEST(RouteClustering, ResultDoesNotDependOnTrajectoryOrder) { + std::mt19937_64 rng(7); + const auto tracks = route_tracks(rng, 4, 6); + std::vector ids(tracks.size()); + std::iota(ids.begin(), ids.end(), 0); + const auto db = make_db(tracks, ids); + + std::vector order(tracks.size()); + std::iota(order.begin(), order.end(), 0); + std::ranges::shuffle(order, rng); + std::vector shuffled; + std::vector shuffled_ids; + for (const auto i : order) { + shuffled.push_back(tracks[i]); + shuffled_ids.push_back(ids[i]); + } + const auto permuted = make_db(shuffled, shuffled_ids); + + const auto a = query::generate_route_clustering(db, {.enabled = true, .min_size = 3}); + const auto b = query::generate_route_clustering(permuted, {.enabled = true, .min_size = 3}); + EXPECT_FALSE(a.reference.members.empty()); + EXPECT_EQ(id_sets(db, a.reference), id_sets(permuted, b.reference)); +} + +TEST(RouteClustering, ReusesAMatchingSelfJoin) { + std::mt19937_64 rng(3); + const auto db = make_db(route_tracks(rng, 3, 5)); + const auto join = query::generate_self_join(db, 500.0); + const auto reused = query::generate_route_clustering( + db, {.enabled = true, .eps_m = 500.0, .min_size = 3}, &join); + const auto fresh = + query::generate_route_clustering(db, {.enabled = true, .eps_m = 500.0, .min_size = 3}); + EXPECT_EQ(reused.candidates, fresh.candidates); + EXPECT_EQ(reused.reference.members, fresh.reference.members); +} + +TEST(RouteClustering, IgnoresASelfJoinOfAnotherDatabaseWithTheSameSize) { + std::mt19937_64 rng(3); + const auto db = make_db(route_tracks(rng, 3, 5)); + const auto other = make_db(std::vector(db.size(), {{39.0, 116.0}, {39.0, 116.001}})); + const auto other_join = query::generate_self_join(other, 500.0); + const query::RouteClusteringConfig config{.enabled = true, .eps_m = 500.0, .min_size = 3}; + const auto workload = query::generate_route_clustering(db, config, &other_join); + const auto fresh = query::generate_route_clustering(db, config); + EXPECT_EQ(workload.candidates, fresh.candidates); + EXPECT_EQ(workload.reference.members, fresh.reference.members); +} + +TEST(RouteClustering, RepeatedPairsCountOnce) { + const auto db = spread_db(3); + const std::vector repeated{{0, 1}, {1, 0}, {0, 1}}; + EXPECT_TRUE(query::cluster_routes(db, repeated, 3).members.empty()); + const std::vector star{{0, 1}, {1, 0}, {0, 2}}; + const auto clusters = query::cluster_routes(db, star, 3); + ASSERT_EQ(clusters.members.size(), 1u); + EXPECT_EQ(clusters.members[0].size(), 3u); +} + +TEST(RouteClustering, UncompressedDatabaseScoresPerfectly) { + std::mt19937_64 rng(11); + const auto db = make_db(route_tracks(rng, 4, 6)); + const auto workload = query::generate_route_clustering(db, {.enabled = true, .min_size = 3}); + ASSERT_FALSE(workload.reference.members.empty()); + const auto scores = evaluator::evaluate_route_clustering(db, workload); + EXPECT_DOUBLE_EQ(scores.score, 1.0); + EXPECT_EQ(scores.predicted_clusters, scores.reference_clusters); +} + +TEST(RouteClustering, CompressedDatabaseIsScoredAgainstTheReference) { + std::mt19937_64 rng(5); + const auto db = make_db(route_tracks(rng, 4, 6)); + const auto workload = query::generate_route_clustering(db, {.enabled = true, .min_size = 3}); + core::Simplification endpoints; + for (std::size_t i = 0; i < db.size(); ++i) { + const std::vector kept{0, db.trajectory(i).size() - 1}; + endpoints.add_trajectory(kept); + } + const auto compressed = core::materialize(db, endpoints); + const auto scores = evaluator::evaluate_route_clustering(compressed, workload); + EXPECT_GE(scores.score, 0.0); + EXPECT_LE(scores.score, 1.0); + const auto direct = evaluator::compare_route_clusters( + workload.reference, evaluator::compressed_route_clusters(compressed, workload)); + EXPECT_DOUBLE_EQ(scores.score, direct.score); +} + +TEST(RouteClustering, CompareScoresOverlapFromBothSides) { + query::RouteClusters reference{{0, 0, 0, 0, -1}, {{0, 1, 2, 3}}}; + query::RouteClusters predicted{{0, 0, -1, -1, -1}, {{0, 1}}}; + const auto scores = evaluator::compare_route_clusters(reference, predicted); + // Both sides' best F1 is 2 * 2 / (4 + 2) + EXPECT_NEAR(scores.score, 2.0 / 3.0, 1e-12); + EXPECT_DOUBLE_EQ(evaluator::compare_route_clusters(reference, reference).score, 1.0); + + const query::RouteClusters none{{-1, -1, -1, -1, -1}, {}}; + EXPECT_DOUBLE_EQ(evaluator::compare_route_clusters(reference, none).score, 0.0); + EXPECT_TRUE(std::isnan(evaluator::compare_route_clusters(none, predicted).score)); +} + +TEST(RouteClustering, RejectsMismatchedDatabasesAndInvalidInput) { + std::mt19937_64 rng(2); + const auto db = make_db(route_tracks(rng, 2, 4)); + const auto workload = query::generate_route_clustering(db, {.enabled = true, .min_size = 3}); + const auto other = make_db(route_tracks(rng, 2, 3)); + EXPECT_THROW(evaluator::evaluate_route_clustering(other, workload), std::invalid_argument); + // Same IDs and order, but a moved endpoint could match outside the cached candidates + std::vector endpoints; + for (std::size_t i = 0; i < db.size(); ++i) { + const auto t = db.trajectory(i); + endpoints.push_back({{t.lat.front(), t.lon.front()}, {t.lat.back(), t.lon.back()}}); + } + EXPECT_NO_THROW(evaluator::evaluate_route_clustering(make_db(endpoints), workload)); + endpoints[0].back().second += 0.01; + EXPECT_THROW(evaluator::evaluate_route_clustering(make_db(endpoints), workload), + std::invalid_argument); + EXPECT_THROW(query::cluster_routes(db, {}, 1), std::invalid_argument); + const std::vector self_pair{{0, 0}}; + EXPECT_THROW(query::cluster_routes(db, self_pair, 3), std::invalid_argument); + EXPECT_THROW(query::generate_route_clustering(db, {.enabled = true, .eps_m = -1.0}), + std::invalid_argument); +} + +TEST(RouteClustering, CombinedIncludesClustering) { + evaluator::Scoretable floor{}; + floor.knn.recall_at_k = 0.0; + floor.range_f1 = 0.0; + floor.where_at.within_radius = 0.0; + floor.join.f1 = 0.0; + floor.clustering.score = 0.5; + auto scores = floor; + scores.knn.recall_at_k = 1.0; + scores.clustering.score = 0.75; + const auto normalized = evaluator::normalize_scores(scores, floor); + EXPECT_DOUBLE_EQ(normalized.clustering, 0.5); + EXPECT_DOUBLE_EQ(normalized.combined, 1.5 / 5.0); + scores.clustering.score = std::numeric_limits::quiet_NaN(); + EXPECT_DOUBLE_EQ(evaluator::normalize_scores(scores, floor).combined, 0.25); +}