Combined score

engine/CMakeLists.txt

@@ -35,6 +35,7 @@ add_library(engine
     src/evaluator/range.cpp
     src/query/join.cpp
     src/evaluator/join.cpp
+    src/evaluator/combined.cpp
 )
 
 target_include_directories(engine PUBLIC include)
@@ -66,6 +67,7 @@ add_executable(engine_tests
     tests/knn_test.cpp
     tests/config_test.cpp
     tests/join_test.cpp
+    tests/combined_test.cpp
 )
 target_link_libraries(engine_tests PRIVATE engine GTest::gtest_main Arrow::arrow_shared Parquet::parquet_shared)
 gtest_discover_tests(engine_tests)

engine/include/backend/engine/evaluator/combined.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "backend/engine/evaluator/evaluator.hpp"
+
+#include <limits>
+
+namespace evaluator {
+
+// 0 = the floor (endpoints only), 1 = perfect. Query types average their own scores and combined
+// averages the types, skipping NaN.
+struct NormalizedScores {
+    static constexpr double missing = std::numeric_limits<double>::quiet_NaN();
+
+    double knn_recall_at_k = missing;
+    double knn_kendalls_tau_topk = missing;
+    double range_recall = missing;
+    double where_at_within_radius = missing;
+    double where_at_average_hit = missing;
+    double join_f1 = missing;
+
+    double knn = missing;
+    double range = missing;
+    double where_at = missing;
+    double join = missing;
+
+    double combined = missing;
+};
+
+// NaN if an input is NaN or the floor is already 1
+double normalize(double score, double floor);
+
+NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &floor);
+
+} // namespace evaluator

engine/src/evaluator/combined.cpp

@@ -0,0 +1,54 @@
+#include "backend/engine/evaluator/combined.hpp"
+
+#include <cmath>
+#include <cstddef>
+#include <initializer_list>
+#include <limits>
+
+namespace evaluator {
+
+namespace {
+
+double mean_ignoring_nan(std::initializer_list<double> values) {
+    double sum = 0.0;
+    std::size_t count = 0;
+    for (const double value : values) {
+        if (!std::isnan(value)) {
+            sum += value;
+            ++count;
+        }
+    }
+    return count == 0 ? std::numeric_limits<double>::quiet_NaN() : sum / static_cast<double>(count);
+}
+
+} // namespace
+
+double normalize(double score, double floor) {
+    if (std::isnan(score) || std::isnan(floor) || floor >= 1.0)
+        return std::numeric_limits<double>::quiet_NaN();
+    return (score - floor) / (1.0 - floor);
+}
+
+NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &floor) {
+    NormalizedScores result;
+    result.knn_recall_at_k = normalize(scores.knn.recall_at_k, floor.knn.recall_at_k);
+    result.knn_kendalls_tau_topk =
+        normalize(scores.knn.kendalls_tau_topk, floor.knn.kendalls_tau_topk);
+    result.range_recall = normalize(scores.range_recall, floor.range_recall);
+    result.where_at_within_radius =
+        normalize(scores.where_at.within_radius, floor.where_at.within_radius);
+    result.where_at_average_hit =
+        normalize(scores.where_at.average_hit, floor.where_at.average_hit);
+    result.join_f1 = normalize(scores.join.f1, floor.join.f1);
+
+    result.knn = mean_ignoring_nan({result.knn_recall_at_k, result.knn_kendalls_tau_topk});
+    result.range = result.range_recall;
+    result.where_at =
+        mean_ignoring_nan({result.where_at_within_radius, result.where_at_average_hit});
+    result.join = result.join_f1;
+
+    result.combined = mean_ignoring_nan({result.knn, result.range, result.where_at, result.join});
+    return result;
+}
+
+} // namespace evaluator

engine/src/pipeline/pipeline.cpp

@@ -2,6 +2,7 @@
 #include "backend/engine/allocation/budget_allocator.hpp"
 #include "backend/engine/core/simplification.hpp"
 #include "backend/engine/core/trajectory_db.hpp"
+#include "backend/engine/evaluator/combined.hpp"
 #include "backend/engine/evaluator/evaluator.hpp"
 #include "backend/engine/io/parquet_loader.hpp"
 #include "backend/engine/io/workload_export.hpp"
@@ -20,6 +21,7 @@
 #include <iostream>
 #include <string>
 #include <string_view>
+#include <vector>
 
 namespace pipeline {
 
@@ -59,6 +61,26 @@ void print_scores(const evaluator::Scoretable &scores) {
               << ", recall: " << scores.join.recall << '\n';
 }
 
+void print_normalized(const evaluator::NormalizedScores &scores) {
+    std::cout << "  normalized: knn=" << scores.knn << ", range=" << scores.range
+              << ", where-at=" << scores.where_at << ", join=" << scores.join
+              << ", combined=" << scores.combined << '\n';
+}
+
+core::Simplification endpoints_only(const core::TrajectoryDb &trajectories) {
+    core::Simplification simplification;
+    for (std::size_t i = 0; i < trajectories.size(); ++i) {
+        const std::size_t n = trajectories.trajectory(i).size();
+        std::vector<std::size_t> kept;
+        if (n > 0)
+            kept.push_back(0);
+        if (n > 1)
+            kept.push_back(n - 1);
+        simplification.add_trajectory(kept);
+    }
+    return simplification;
+}
+
 } // namespace
 
 void run(const Config &config) {
@@ -86,6 +108,17 @@ void run(const Config &config) {
         std::cout << "workloads written to " << config.export_dir.string() << '\n';
     }
 
+    const evaluator::Workloads workloads{
+        .knn = &knn_workload, .where_at = where_at.queries, .range = range.queries, .join = &join};
+
+    const auto floor_db = core::materialize(database, endpoints_only(database));
+    std::cout << "floor, endpoints only: ";
+    print_size("compressed", floor_db);
+    const auto floor = evaluator::Evaluator(database, floor_db, workloads, config.where_at_scoring)
+                           .evaluate()
+                           .scores;
+    print_scores(floor);
+
     for (const double share : config.budget_shares) {
         const auto budgets = allocation::create_trajectory_budgets(
             database, allocation::AllocatorType::SizeProportional, total_budget(database, share));
@@ -96,14 +129,11 @@ void run(const Config &config) {
             std::cout << share * 100 << "% " << magic_enum::enum_name(algorithm) << ": ";
             print_size("compressed", compressed);
 
-            const evaluator::Evaluator evaluator(database, compressed,
-                                                 {.knn = &knn_workload,
-                                                  .where_at = where_at.queries,
-                                                  .range = range.queries,
-                                                  .join = &join},
+            const evaluator::Evaluator evaluator(database, compressed, workloads,
                                                  config.where_at_scoring);
             const auto evaluation = evaluator.evaluate();
             print_scores(evaluation.scores);
+            print_normalized(evaluator::normalize_scores(evaluation.scores, floor));
 
             if (config.export_results) {
                 const std::string name =

engine/tests/combined_test.cpp

@@ -0,0 +1,81 @@
+#include "backend/engine/evaluator/combined.hpp"
+
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <limits>
+
+namespace {
+
+constexpr double missing = std::numeric_limits<double>::quiet_NaN();
+
+evaluator::Scoretable all_scores(double value) {
+    evaluator::Scoretable scores{};
+    scores.knn.recall_at_k = value;
+    scores.knn.kendalls_tau_topk = value;
+    scores.range_recall = value;
+    scores.where_at.within_radius = value;
+    scores.where_at.average_hit = value;
+    scores.join.f1 = value;
+    return scores;
+}
+
+} // namespace
+
+TEST(Normalize, MapsFloorToZeroAndPerfectToOne) {
+    EXPECT_DOUBLE_EQ(evaluator::normalize(0.2, 0.2), 0.0);
+    EXPECT_DOUBLE_EQ(evaluator::normalize(1.0, 0.2), 1.0);
+    EXPECT_DOUBLE_EQ(evaluator::normalize(0.6, 0.2), 0.5);
+}
+
+TEST(Normalize, StretchesMetricsWithAHighFloor) {
+    EXPECT_DOUBLE_EQ(evaluator::normalize(0.3, 0.2), 0.125);
+    EXPECT_DOUBLE_EQ(evaluator::normalize(0.9, 0.8), 0.5);
+}
+
+TEST(Normalize, IsNegativeBelowFloor) { EXPECT_DOUBLE_EQ(evaluator::normalize(0.1, 0.2), -0.125); }
+
+TEST(Normalize, IsNanWithoutRoomOrInput) {
+    EXPECT_TRUE(std::isnan(evaluator::normalize(1.0, 1.0)));
+    EXPECT_TRUE(std::isnan(evaluator::normalize(missing, 0.5)));
+    EXPECT_TRUE(std::isnan(evaluator::normalize(0.5, missing)));
+}
+
+TEST(NormalizeScores, FloorScoresZeroAndPerfectScoresOne) {
+    const auto floor = all_scores(0.4);
+    EXPECT_DOUBLE_EQ(evaluator::normalize_scores(floor, floor).combined, 0.0);
+    EXPECT_DOUBLE_EQ(evaluator::normalize_scores(all_scores(1.0), floor).combined, 1.0);
+}
+
+TEST(NormalizeScores, WeighsQueryTypesEquallyNotScores) {
+    const auto floor = all_scores(0.0);
+    auto scores = all_scores(0.0);
+    scores.knn.recall_at_k = 1.0;
+    scores.range_recall = 1.0;
+    const auto normalized = evaluator::normalize_scores(scores, floor);
+    EXPECT_DOUBLE_EQ(normalized.knn_recall_at_k, 1.0);
+    EXPECT_DOUBLE_EQ(normalized.knn, 0.5);
+    EXPECT_DOUBLE_EQ(normalized.range, 1.0);
+    EXPECT_DOUBLE_EQ(normalized.where_at, 0.0);
+    EXPECT_DOUBLE_EQ(normalized.join, 0.0);
+    // (0.5 + 1 + 0 + 0) / 4, not a flat 2 / 6 over the scores
+    EXPECT_DOUBLE_EQ(normalized.combined, 0.375);
+}
+
+TEST(NormalizeScores, SkipsMissingQueryTypes) {
+    auto floor = all_scores(0.5);
+    auto scores = all_scores(0.75);
+    floor.join.f1 = missing;
+    scores.join.f1 = missing;
+    scores.where_at.average_hit = missing;
+    const auto normalized = evaluator::normalize_scores(scores, floor);
+    EXPECT_TRUE(std::isnan(normalized.join));
+    EXPECT_DOUBLE_EQ(normalized.where_at, 0.5);
+    EXPECT_DOUBLE_EQ(normalized.combined, 0.5);
+}
+
+TEST(NormalizeScores, AllMissingIsNan) {
+    const auto normalized = evaluator::normalize_scores(all_scores(missing), all_scores(missing));
+    EXPECT_TRUE(std::isnan(normalized.knn));
+    EXPECT_TRUE(std::isnan(normalized.combined));
+}