diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index e46bdf7..fdeab7d 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -35,6 +35,7 @@ add_library(engine src/evaluator/range.cpp src/query/join.cpp src/evaluator/join.cpp + src/evaluator/combined.cpp ) target_include_directories(engine PUBLIC include) @@ -66,6 +67,7 @@ add_executable(engine_tests tests/knn_test.cpp tests/config_test.cpp tests/join_test.cpp + tests/combined_test.cpp ) target_link_libraries(engine_tests PRIVATE engine GTest::gtest_main Arrow::arrow_shared Parquet::parquet_shared) gtest_discover_tests(engine_tests) diff --git a/engine/include/backend/engine/evaluator/combined.hpp b/engine/include/backend/engine/evaluator/combined.hpp new file mode 100644 index 0000000..399b60a --- /dev/null +++ b/engine/include/backend/engine/evaluator/combined.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include "backend/engine/evaluator/evaluator.hpp" + +#include + +namespace evaluator { + +// 0 = the floor (endpoints only), 1 = perfect. Query types average their own scores and combined +// averages the types, skipping NaN. +struct NormalizedScores { + static constexpr double missing = std::numeric_limits::quiet_NaN(); + + double knn_recall_at_k = missing; + double knn_kendalls_tau_topk = missing; + double range_recall = missing; + double where_at_within_radius = missing; + double where_at_average_hit = missing; + double join_f1 = missing; + + double knn = missing; + double range = missing; + double where_at = missing; + double join = missing; + + double combined = missing; +}; + +// NaN if an input is NaN or the floor is already 1 +double normalize(double score, double floor); + +NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &floor); + +} // namespace evaluator diff --git a/engine/src/evaluator/combined.cpp b/engine/src/evaluator/combined.cpp new file mode 100644 index 0000000..fa60bc4 --- /dev/null +++ b/engine/src/evaluator/combined.cpp @@ -0,0 +1,54 @@ +#include "backend/engine/evaluator/combined.hpp" + +#include +#include +#include +#include + +namespace evaluator { + +namespace { + +double mean_ignoring_nan(std::initializer_list values) { + double sum = 0.0; + std::size_t count = 0; + for (const double value : values) { + if (!std::isnan(value)) { + sum += value; + ++count; + } + } + return count == 0 ? std::numeric_limits::quiet_NaN() : sum / static_cast(count); +} + +} // namespace + +double normalize(double score, double floor) { + if (std::isnan(score) || std::isnan(floor) || floor >= 1.0) + return std::numeric_limits::quiet_NaN(); + return (score - floor) / (1.0 - floor); +} + +NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &floor) { + NormalizedScores result; + result.knn_recall_at_k = normalize(scores.knn.recall_at_k, floor.knn.recall_at_k); + result.knn_kendalls_tau_topk = + normalize(scores.knn.kendalls_tau_topk, floor.knn.kendalls_tau_topk); + result.range_recall = normalize(scores.range_recall, floor.range_recall); + result.where_at_within_radius = + normalize(scores.where_at.within_radius, floor.where_at.within_radius); + result.where_at_average_hit = + normalize(scores.where_at.average_hit, floor.where_at.average_hit); + result.join_f1 = normalize(scores.join.f1, floor.join.f1); + + result.knn = mean_ignoring_nan({result.knn_recall_at_k, result.knn_kendalls_tau_topk}); + result.range = result.range_recall; + result.where_at = + mean_ignoring_nan({result.where_at_within_radius, result.where_at_average_hit}); + result.join = result.join_f1; + + result.combined = mean_ignoring_nan({result.knn, result.range, result.where_at, result.join}); + return result; +} + +} // namespace evaluator diff --git a/engine/src/pipeline/pipeline.cpp b/engine/src/pipeline/pipeline.cpp index d45222d..98d05e5 100644 --- a/engine/src/pipeline/pipeline.cpp +++ b/engine/src/pipeline/pipeline.cpp @@ -2,6 +2,7 @@ #include "backend/engine/allocation/budget_allocator.hpp" #include "backend/engine/core/simplification.hpp" #include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/evaluator/combined.hpp" #include "backend/engine/evaluator/evaluator.hpp" #include "backend/engine/io/parquet_loader.hpp" #include "backend/engine/io/workload_export.hpp" @@ -20,6 +21,7 @@ #include #include #include +#include namespace pipeline { @@ -59,6 +61,26 @@ void print_scores(const evaluator::Scoretable &scores) { << ", recall: " << scores.join.recall << '\n'; } +void print_normalized(const evaluator::NormalizedScores &scores) { + std::cout << " normalized: knn=" << scores.knn << ", range=" << scores.range + << ", where-at=" << scores.where_at << ", join=" << scores.join + << ", combined=" << scores.combined << '\n'; +} + +core::Simplification endpoints_only(const core::TrajectoryDb &trajectories) { + core::Simplification simplification; + for (std::size_t i = 0; i < trajectories.size(); ++i) { + const std::size_t n = trajectories.trajectory(i).size(); + std::vector kept; + if (n > 0) + kept.push_back(0); + if (n > 1) + kept.push_back(n - 1); + simplification.add_trajectory(kept); + } + return simplification; +} + } // namespace void run(const Config &config) { @@ -86,6 +108,17 @@ void run(const Config &config) { std::cout << "workloads written to " << config.export_dir.string() << '\n'; } + const evaluator::Workloads workloads{ + .knn = &knn_workload, .where_at = where_at.queries, .range = range.queries, .join = &join}; + + const auto floor_db = core::materialize(database, endpoints_only(database)); + std::cout << "floor, endpoints only: "; + print_size("compressed", floor_db); + const auto floor = evaluator::Evaluator(database, floor_db, workloads, config.where_at_scoring) + .evaluate() + .scores; + print_scores(floor); + for (const double share : config.budget_shares) { const auto budgets = allocation::create_trajectory_budgets( database, allocation::AllocatorType::SizeProportional, total_budget(database, share)); @@ -96,14 +129,11 @@ void run(const Config &config) { std::cout << share * 100 << "% " << magic_enum::enum_name(algorithm) << ": "; print_size("compressed", compressed); - const evaluator::Evaluator evaluator(database, compressed, - {.knn = &knn_workload, - .where_at = where_at.queries, - .range = range.queries, - .join = &join}, + const evaluator::Evaluator evaluator(database, compressed, workloads, config.where_at_scoring); const auto evaluation = evaluator.evaluate(); print_scores(evaluation.scores); + print_normalized(evaluator::normalize_scores(evaluation.scores, floor)); if (config.export_results) { const std::string name = diff --git a/engine/tests/combined_test.cpp b/engine/tests/combined_test.cpp new file mode 100644 index 0000000..b91c515 --- /dev/null +++ b/engine/tests/combined_test.cpp @@ -0,0 +1,81 @@ +#include "backend/engine/evaluator/combined.hpp" + +#include + +#include +#include + +namespace { + +constexpr double missing = std::numeric_limits::quiet_NaN(); + +evaluator::Scoretable all_scores(double value) { + evaluator::Scoretable scores{}; + scores.knn.recall_at_k = value; + scores.knn.kendalls_tau_topk = value; + scores.range_recall = value; + scores.where_at.within_radius = value; + scores.where_at.average_hit = value; + scores.join.f1 = value; + return scores; +} + +} // namespace + +TEST(Normalize, MapsFloorToZeroAndPerfectToOne) { + EXPECT_DOUBLE_EQ(evaluator::normalize(0.2, 0.2), 0.0); + EXPECT_DOUBLE_EQ(evaluator::normalize(1.0, 0.2), 1.0); + EXPECT_DOUBLE_EQ(evaluator::normalize(0.6, 0.2), 0.5); +} + +TEST(Normalize, StretchesMetricsWithAHighFloor) { + EXPECT_DOUBLE_EQ(evaluator::normalize(0.3, 0.2), 0.125); + EXPECT_DOUBLE_EQ(evaluator::normalize(0.9, 0.8), 0.5); +} + +TEST(Normalize, IsNegativeBelowFloor) { EXPECT_DOUBLE_EQ(evaluator::normalize(0.1, 0.2), -0.125); } + +TEST(Normalize, IsNanWithoutRoomOrInput) { + EXPECT_TRUE(std::isnan(evaluator::normalize(1.0, 1.0))); + EXPECT_TRUE(std::isnan(evaluator::normalize(missing, 0.5))); + EXPECT_TRUE(std::isnan(evaluator::normalize(0.5, missing))); +} + +TEST(NormalizeScores, FloorScoresZeroAndPerfectScoresOne) { + const auto floor = all_scores(0.4); + EXPECT_DOUBLE_EQ(evaluator::normalize_scores(floor, floor).combined, 0.0); + EXPECT_DOUBLE_EQ(evaluator::normalize_scores(all_scores(1.0), floor).combined, 1.0); +} + +TEST(NormalizeScores, WeighsQueryTypesEquallyNotScores) { + const auto floor = all_scores(0.0); + auto scores = all_scores(0.0); + scores.knn.recall_at_k = 1.0; + scores.range_recall = 1.0; + const auto normalized = evaluator::normalize_scores(scores, floor); + EXPECT_DOUBLE_EQ(normalized.knn_recall_at_k, 1.0); + EXPECT_DOUBLE_EQ(normalized.knn, 0.5); + EXPECT_DOUBLE_EQ(normalized.range, 1.0); + EXPECT_DOUBLE_EQ(normalized.where_at, 0.0); + EXPECT_DOUBLE_EQ(normalized.join, 0.0); + // (0.5 + 1 + 0 + 0) / 4, not a flat 2 / 6 over the scores + EXPECT_DOUBLE_EQ(normalized.combined, 0.375); +} + +TEST(NormalizeScores, SkipsMissingQueryTypes) { + auto floor = all_scores(0.5); + auto scores = all_scores(0.75); + floor.join.f1 = missing; + scores.join.f1 = missing; + scores.where_at.average_hit = missing; + const auto normalized = evaluator::normalize_scores(scores, floor); + EXPECT_TRUE(std::isnan(normalized.join)); + EXPECT_DOUBLE_EQ(normalized.where_at, 0.5); + EXPECT_DOUBLE_EQ(normalized.combined, 0.5); +} + +TEST(NormalizeScores, AllMissingIsNan) { + const auto normalized = evaluator::normalize_scores(all_scores(missing), all_scores(missing)); + EXPECT_TRUE(std::isnan(normalized.knn)); + EXPECT_TRUE(std::isnan(normalized.combined)); +}