From dbc1b9cc6f4af8e4e84531f544f09a9c053f1608 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 1 Sep 2026 11:07:00 +0800 Subject: [PATCH 1/4] io_pattern frame --- mooncake-store/include/io_pattern/analyzer.h | 14 ++ mooncake-store/include/io_pattern/collector.h | 17 ++ .../include/io_pattern/io_pattern.h | 8 + mooncake-store/include/io_pattern/ops.h | 44 ++++ .../include/io_pattern/policy_engine.h | 23 ++ mooncake-store/include/io_pattern/registry.h | 68 ++++++ mooncake-store/include/io_pattern/types.h | 216 ++++++++++++++++++ mooncake-store/tests/CMakeLists.txt | 1 + .../tests/io_pattern_framework_test.cpp | 59 +++++ 9 files changed, 450 insertions(+) create mode 100644 mooncake-store/include/io_pattern/analyzer.h create mode 100644 mooncake-store/include/io_pattern/collector.h create mode 100644 mooncake-store/include/io_pattern/io_pattern.h create mode 100644 mooncake-store/include/io_pattern/ops.h create mode 100644 mooncake-store/include/io_pattern/policy_engine.h create mode 100644 mooncake-store/include/io_pattern/registry.h create mode 100644 mooncake-store/include/io_pattern/types.h create mode 100644 mooncake-store/tests/io_pattern_framework_test.cpp diff --git a/mooncake-store/include/io_pattern/analyzer.h b/mooncake-store/include/io_pattern/analyzer.h new file mode 100644 index 0000000000..94efd1f71e --- /dev/null +++ b/mooncake-store/include/io_pattern/analyzer.h @@ -0,0 +1,14 @@ +#pragma once + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +class IoPatternAnalyzer { + public: + virtual ~IoPatternAnalyzer() = default; + + virtual PatternResult Analyze(const IoPatternSnapshot& snapshot) const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector.h b/mooncake-store/include/io_pattern/collector.h new file mode 100644 index 0000000000..94d1ccbbee --- /dev/null +++ b/mooncake-store/include/io_pattern/collector.h @@ -0,0 +1,17 @@ +#pragma once + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +class IoPatternCollector { + public: + virtual ~IoPatternCollector() = default; + + virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; + virtual void RecordAccess(const AccessRecord& record) = 0; + virtual void RecordStorageMetric(const StorageMetric& metric) = 0; + virtual IoPatternSnapshot GetSnapshot() const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h new file mode 100644 index 0000000000..99834c3561 --- /dev/null +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -0,0 +1,8 @@ +#pragma once + +#include "io_pattern/analyzer.h" +#include "io_pattern/collector.h" +#include "io_pattern/ops.h" +#include "io_pattern/policy_engine.h" +#include "io_pattern/registry.h" +#include "io_pattern/types.h" diff --git a/mooncake-store/include/io_pattern/ops.h b/mooncake-store/include/io_pattern/ops.h new file mode 100644 index 0000000000..5db67f2953 --- /dev/null +++ b/mooncake-store/include/io_pattern/ops.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include "io_pattern/types.h" +#include "types.h" + +namespace mooncake::io_pattern { + +class EvictionOps { + public: + virtual ~EvictionOps() = default; + + virtual EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const = 0; +}; + +class PrefetchOps { + public: + virtual ~PrefetchOps() = default; + + virtual PrefetchPlan Evaluate(const PolicyContext& context, + const TraceHistory& trace) const = 0; +}; + +class AdmissionOps { + public: + virtual ~AdmissionOps() = default; + + virtual AdmissionResult Evaluate(const ObjectRef& object, + CacheTier target_tier, + const PolicyContext& context) const = 0; +}; + +// Data movement is a client-side seam. Keeping it separate prevents a +// SubMaster-side prefetch planner from depending on a concrete storage client. +class PrefetchExecutor { + public: + virtual ~PrefetchExecutor() = default; + + virtual ErrorCode Execute(const PrefetchPlan& plan) = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h new file mode 100644 index 0000000000..bf7e2adc19 --- /dev/null +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +class PolicyEngine { + public: + virtual ~PolicyEngine() = default; + + virtual EvictionPlan PlanEviction(const PolicyContext& context, + CacheTier tier, + uint64_t target_bytes) const = 0; + virtual PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const = 0; + virtual AdmissionResult DecideAdmission( + const ObjectRef& object, CacheTier target_tier, + const PolicyContext& context) const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/registry.h b/mooncake-store/include/io_pattern/registry.h new file mode 100644 index 0000000000..e560967310 --- /dev/null +++ b/mooncake-store/include/io_pattern/registry.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io_pattern/ops.h" + +namespace mooncake::io_pattern { + +template +class OpsRegistry { + public: + using Factory = std::function()>; + + bool Register(std::string name, Factory factory) { + if (name.empty() || !factory) { + return false; + } + std::unique_lock lock(mutex_); + return factories_.emplace(std::move(name), std::move(factory)).second; + } + + std::shared_ptr Create(std::string_view name) const { + Factory factory; + { + std::shared_lock lock(mutex_); + const auto it = factories_.find(std::string(name)); + if (it == factories_.end()) { + return nullptr; + } + factory = it->second; + } + return factory(); + } + + std::vector RegisteredNames() const { + std::vector names; + { + std::shared_lock lock(mutex_); + names.reserve(factories_.size()); + for (const auto& entry : factories_) { + names.push_back(entry.first); + } + } + std::sort(names.begin(), names.end()); + return names; + } + + private: + mutable std::shared_mutex mutex_; + std::unordered_map factories_; +}; + +struct PolicyOpsRegistries { + OpsRegistry eviction; + OpsRegistry prefetch; + OpsRegistry admission; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h new file mode 100644 index 0000000000..2a39ae0cc7 --- /dev/null +++ b/mooncake-store/include/io_pattern/types.h @@ -0,0 +1,216 @@ +#pragma once + +#include +#include +#include + +#include "tenant_id.h" + +namespace mooncake::io_pattern { + +enum class CacheTier : uint8_t { + kL0Hbm = 0, + kL1Host = 1, + kL2Segment = 2, + kL3NofSsd = 3, +}; + +using CacheTierMask = uint8_t; + +constexpr CacheTierMask CacheTierBit(CacheTier tier) { + return static_cast(1U << static_cast(tier)); +} + +enum class IoOperation : uint8_t { + kGet, + kPut, + kTierUp, + kTierDown, +}; + +enum class CacheLayout : uint8_t { + kUnknown, + kLayerFirst, + kPageFirst, + kPageFirstDirect, + kPageBased, + kHmaMultiGroup, +}; + +enum class StorageGcState : uint8_t { + kUnknown, + kIdle, + kRunning, + kStalled, +}; + +enum class WorkloadType : uint8_t { + kUnknown, + kCodeAgent, + kGenerativeRecommendation, + kMultiTurnConversation, + kMixed, +}; + +struct ObjectRef { + TenantId tenant_id; + std::string key; + + bool operator==(const ObjectRef&) const = default; +}; + +struct InferenceMetrics { + ObjectRef object; + std::string session_id; + CacheLayout layout{CacheLayout::kUnknown}; + uint32_t layout_group{0}; + uint32_t prefix_depth{0}; + uint32_t prefix_fanout{0}; + uint32_t match_length{0}; + uint32_t continuous_prefix_length{0}; + uint32_t token_count{0}; + float recompute_cost{0.0F}; + uint8_t request_priority{0}; +}; + +struct AccessRecord { + ObjectRef object; + uint64_t observed_at_ns{0}; + uint64_t block_size{0}; + uint64_t latency_us{0}; + CacheTier tier{CacheTier::kL2Segment}; + IoOperation operation{IoOperation::kGet}; + bool is_hit{false}; +}; + +struct StorageMetric { + std::string source_id; + uint64_t observed_at_ns{0}; + CacheTier tier{CacheTier::kL2Segment}; + StorageGcState gc_state{StorageGcState::kUnknown}; + uint64_t read_bandwidth_bytes_per_sec{0}; + uint64_t write_bandwidth_bytes_per_sec{0}; + uint64_t read_latency_us{0}; + uint64_t write_latency_us{0}; + uint64_t used_bytes{0}; + uint64_t capacity_bytes{0}; + uint64_t rpc_latency_us{0}; + float memory_used_ratio{0.0F}; +}; + +struct KeyMetrics { + ObjectRef object; + uint64_t last_access_time_ns{0}; + uint64_t access_count_window{0}; + uint64_t idle_time_us{0}; + uint64_t block_size{0}; + uint64_t transfer_eta_us{0}; + uint32_t token_count{0}; + uint32_t prefix_depth{0}; + uint32_t prefix_fanout{0}; + uint32_t match_length{0}; + uint32_t continuous_prefix_length{0}; + uint32_t other_replica_count{0}; + uint32_t write_batch_size{0}; + uint32_t write_frequency{0}; + uint64_t write_object_size{0}; + float recompute_cost{0.0F}; + float overwrite_ratio{0.0F}; + CacheTierMask replica_tiers{0}; + CacheLayout layout{CacheLayout::kUnknown}; + uint32_t layout_group{0}; + uint8_t request_priority{0}; + bool active{false}; + bool pinned{false}; + bool ssd_replica_exists{false}; + bool write_burst{false}; +}; + +struct IoPatternSnapshot { + uint64_t generated_at_ns{0}; + std::vector keys; + std::vector storage; +}; + +struct KeyPattern { + ObjectRef object; + float confidence{0.0F}; + float frequency_score{0.0F}; + float idle_score{0.0F}; + float prefix_score{0.0F}; + float recompute_score{0.0F}; + float transfer_roi{0.0F}; + bool migration_safe{false}; +}; + +struct PatternResult { + WorkloadType workload_type{WorkloadType::kUnknown}; + float workload_confidence{0.0F}; + std::vector keys; +}; + +struct PolicyContext { + IoPatternSnapshot snapshot; + PatternResult analysis; +}; + +struct TraceEvent { + ObjectRef object; + uint64_t observed_at_ns{0}; + uint32_t match_length{0}; + bool is_hit{false}; +}; + +struct TraceHistory { + std::vector events; +}; + +enum class PrefetchStrategy : uint8_t { + kBestEffort, + kTimeout, + kWaitComplete, +}; + +struct PrefetchCandidate { + ObjectRef object; + CacheTier source_tier{CacheTier::kL3NofSsd}; + CacheTier target_tier{CacheTier::kL2Segment}; + uint64_t bytes{0}; + float priority{0.0F}; + float confidence{0.0F}; +}; + +struct PrefetchPlan { + PrefetchStrategy strategy{PrefetchStrategy::kBestEffort}; + uint64_t timeout_us{0}; + std::vector candidates; +}; + +struct EvictionCandidate { + ObjectRef object; + uint64_t bytes{0}; + float score{0.0F}; +}; + +struct EvictionPlan { + CacheTier source_tier{CacheTier::kL0Hbm}; + uint64_t target_bytes{0}; + std::vector candidates; +}; + +enum class AdmissionDecision : uint8_t { + kAdmit, + kRejectFrequency, + kRejectWatermark, + kRejectPrefix, + kDefer, +}; + +struct AdmissionResult { + ObjectRef object; + CacheTier target_tier{CacheTier::kL2Segment}; + AdmissionDecision decision{AdmissionDecision::kDefer}; + float confidence{0.0F}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index 303be02254..d148c51f36 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -45,6 +45,7 @@ add_test( set_tests_properties(replica_selection_env_opt_in_test PROPERTIES ENVIRONMENT "MC_STORE_REPLICA_SCORING=1") add_store_test(eviction_strategy_test eviction_strategy_test.cpp) +add_store_test(io_pattern_framework_test io_pattern_framework_test.cpp) add_store_test(deadline_scheduler_test deadline_scheduler_test.cpp) add_store_test(kv_event_publisher_test kv_event_publisher_test.cpp) if(ENABLE_KV_EVENTS) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp new file mode 100644 index 0000000000..cd74468a20 --- /dev/null +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -0,0 +1,59 @@ +#include "io_pattern/io_pattern.h" + +#include +#include + +#include + +namespace mooncake::io_pattern { +namespace { + +class TestEvictionOps final : public EvictionOps { + public: + EvictionPlan Evaluate(const PolicyContext&, CacheTier tier, + uint64_t target_bytes) const override { + return EvictionPlan{.source_tier = tier, + .target_bytes = target_bytes, + .candidates = {}}; + } +}; + +TEST(IoPatternFrameworkTest, PublicSeamsRemainAbstract) { + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); +} + +TEST(IoPatternFrameworkTest, CacheTierMaskRepresentsAllTiers) { + const CacheTierMask all_tiers = + CacheTierBit(CacheTier::kL0Hbm) | CacheTierBit(CacheTier::kL1Host) | + CacheTierBit(CacheTier::kL2Segment) | + CacheTierBit(CacheTier::kL3NofSsd); + + EXPECT_EQ(all_tiers, 0x0F); +} + +TEST(IoPatternFrameworkTest, RegistryCreatesTypedOpsAndRejectsDuplicates) { + OpsRegistry registry; + + EXPECT_TRUE(registry.Register( + "test", [] { return std::make_shared(); })); + EXPECT_FALSE(registry.Register( + "test", [] { return std::make_shared(); })); + EXPECT_FALSE(registry.Register("", {})); + EXPECT_EQ(registry.RegisteredNames(), std::vector{"test"}); + + const auto ops = registry.Create("test"); + ASSERT_NE(ops, nullptr); + const auto plan = ops->Evaluate({}, CacheTier::kL2Segment, 4096); + EXPECT_EQ(plan.source_tier, CacheTier::kL2Segment); + EXPECT_EQ(plan.target_bytes, 4096); + EXPECT_EQ(registry.Create("missing"), nullptr); +} + +} // namespace +} // namespace mooncake::io_pattern From 2a8bc331ca4eb0ee2a2b1f878dc95da53a42fb50 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 1 Sep 2026 11:52:28 +0800 Subject: [PATCH 2/4] io_pattern frame --- mooncake-store/include/admission_ops.h | 3 + mooncake-store/include/cache_view_manager.h | 3 + mooncake-store/include/cfm_client.h | 3 + mooncake-store/include/eviction_ops.h | 3 + mooncake-store/include/io_pattern.h | 3 + mooncake-store/include/io_pattern/analyzer.h | 5 + mooncake-store/include/io_pattern/client.h | 18 ++ mooncake-store/include/io_pattern/collector.h | 2 + .../include/io_pattern/io_pattern.h | 2 + mooncake-store/include/io_pattern/ops.h | 5 +- .../include/io_pattern/policy_engine.h | 48 +++++ mooncake-store/include/io_pattern/registry.h | 1 + mooncake-store/include/io_pattern/types.h | 35 +++- .../include/io_pattern/view_manager.h | 17 ++ mooncake-store/include/io_pattern_analyzer.h | 3 + mooncake-store/include/io_pattern_collector.h | 3 + mooncake-store/include/io_pattern_registry.h | 3 + mooncake-store/include/io_pattern_types.h | 3 + mooncake-store/include/policy_engine.h | 3 + mooncake-store/include/prefetch_ops.h | 3 + .../tests/io_pattern_framework_test.cpp | 187 ++++++++++++++++++ 21 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 mooncake-store/include/admission_ops.h create mode 100644 mooncake-store/include/cache_view_manager.h create mode 100644 mooncake-store/include/cfm_client.h create mode 100644 mooncake-store/include/eviction_ops.h create mode 100644 mooncake-store/include/io_pattern.h create mode 100644 mooncake-store/include/io_pattern/client.h create mode 100644 mooncake-store/include/io_pattern/view_manager.h create mode 100644 mooncake-store/include/io_pattern_analyzer.h create mode 100644 mooncake-store/include/io_pattern_collector.h create mode 100644 mooncake-store/include/io_pattern_registry.h create mode 100644 mooncake-store/include/io_pattern_types.h create mode 100644 mooncake-store/include/policy_engine.h create mode 100644 mooncake-store/include/prefetch_ops.h diff --git a/mooncake-store/include/admission_ops.h b/mooncake-store/include/admission_ops.h new file mode 100644 index 0000000000..9ef1ade825 --- /dev/null +++ b/mooncake-store/include/admission_ops.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/ops.h" diff --git a/mooncake-store/include/cache_view_manager.h b/mooncake-store/include/cache_view_manager.h new file mode 100644 index 0000000000..0a96be6fec --- /dev/null +++ b/mooncake-store/include/cache_view_manager.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/view_manager.h" diff --git a/mooncake-store/include/cfm_client.h b/mooncake-store/include/cfm_client.h new file mode 100644 index 0000000000..c94a0b0929 --- /dev/null +++ b/mooncake-store/include/cfm_client.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/client.h" diff --git a/mooncake-store/include/eviction_ops.h b/mooncake-store/include/eviction_ops.h new file mode 100644 index 0000000000..9ef1ade825 --- /dev/null +++ b/mooncake-store/include/eviction_ops.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/ops.h" diff --git a/mooncake-store/include/io_pattern.h b/mooncake-store/include/io_pattern.h new file mode 100644 index 0000000000..b382ec908e --- /dev/null +++ b/mooncake-store/include/io_pattern.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/io_pattern.h" diff --git a/mooncake-store/include/io_pattern/analyzer.h b/mooncake-store/include/io_pattern/analyzer.h index 94efd1f71e..1b5e8b6682 100644 --- a/mooncake-store/include/io_pattern/analyzer.h +++ b/mooncake-store/include/io_pattern/analyzer.h @@ -4,11 +4,16 @@ namespace mooncake::io_pattern { +// Converts an immutable snapshot into workload and per-object features. class IoPatternAnalyzer { public: virtual ~IoPatternAnalyzer() = default; virtual PatternResult Analyze(const IoPatternSnapshot& snapshot) const = 0; + virtual WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const = 0; + virtual float CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const = 0; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/client.h b/mooncake-store/include/io_pattern/client.h new file mode 100644 index 0000000000..1c4e061c9d --- /dev/null +++ b/mooncake-store/include/io_pattern/client.h @@ -0,0 +1,18 @@ +#pragma once + +#include "../types.h" +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +// Adapter seam between an inference node and the remote Cache Flow Manager. +class CfmClient { + public: + virtual ~CfmClient() = default; + + virtual ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) = 0; + virtual ErrorCode ReceivePolicy(const PolicyCommand& command) = 0; + virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector.h b/mooncake-store/include/io_pattern/collector.h index 94d1ccbbee..986a477b8f 100644 --- a/mooncake-store/include/io_pattern/collector.h +++ b/mooncake-store/include/io_pattern/collector.h @@ -4,10 +4,12 @@ namespace mooncake::io_pattern { +// Collects non-blocking, already-aggregated observations from data paths. class IoPatternCollector { public: virtual ~IoPatternCollector() = default; + // Implementations must not block the caller on RPC or storage I/O. virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; virtual void RecordAccess(const AccessRecord& record) = 0; virtual void RecordStorageMetric(const StorageMetric& metric) = 0; diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 99834c3561..4ffbcaa5f1 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -1,8 +1,10 @@ #pragma once #include "io_pattern/analyzer.h" +#include "io_pattern/client.h" #include "io_pattern/collector.h" #include "io_pattern/ops.h" #include "io_pattern/policy_engine.h" #include "io_pattern/registry.h" #include "io_pattern/types.h" +#include "io_pattern/view_manager.h" diff --git a/mooncake-store/include/io_pattern/ops.h b/mooncake-store/include/io_pattern/ops.h index 5db67f2953..37e6c64941 100644 --- a/mooncake-store/include/io_pattern/ops.h +++ b/mooncake-store/include/io_pattern/ops.h @@ -3,10 +3,11 @@ #include #include "io_pattern/types.h" -#include "types.h" +#include "../types.h" namespace mooncake::io_pattern { +// Produces an eviction plan; it does not move or delete data. class EvictionOps { public: virtual ~EvictionOps() = default; @@ -15,6 +16,7 @@ class EvictionOps { uint64_t target_bytes) const = 0; }; +// Produces a prefetch plan; execution belongs to PrefetchExecutor. class PrefetchOps { public: virtual ~PrefetchOps() = default; @@ -23,6 +25,7 @@ class PrefetchOps { const TraceHistory& trace) const = 0; }; +// Decides whether an object may enter a target tier. class AdmissionOps { public: virtual ~AdmissionOps() = default; diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index bf7e2adc19..4d419b1a72 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -1,11 +1,15 @@ #pragma once #include +#include +#include +#include "io_pattern/ops.h" #include "io_pattern/types.h" namespace mooncake::io_pattern { +// Coordinates configured Ops implementations without owning data-path state. class PolicyEngine { public: virtual ~PolicyEngine() = default; @@ -20,4 +24,48 @@ class PolicyEngine { const PolicyContext& context) const = 0; }; +// A small composition adapter that wires selected Ops instances together. +// Missing optional Ops degrade to empty plans or a deferred admission result. +class ComposedPolicyEngine final : public PolicyEngine { + public: + ComposedPolicyEngine(std::shared_ptr eviction, + std::shared_ptr prefetch, + std::shared_ptr admission) + : eviction_(std::move(eviction)), + prefetch_(std::move(prefetch)), + admission_(std::move(admission)) {} + + EvictionPlan PlanEviction(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override { + if (!eviction_) { + return EvictionPlan{.source_tier = tier, + .target_bytes = target_bytes}; + } + return eviction_->Evaluate(context, tier, target_bytes); + } + + PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const override { + if (!prefetch_) { + return {}; + } + return prefetch_->Evaluate(context, trace); + } + + AdmissionResult DecideAdmission(const ObjectRef& object, + CacheTier target_tier, + const PolicyContext& context) const override { + if (!admission_) { + return AdmissionResult{.object = object, + .target_tier = target_tier}; + } + return admission_->Evaluate(object, target_tier, context); + } + + private: + std::shared_ptr eviction_; + std::shared_ptr prefetch_; + std::shared_ptr admission_; +}; + } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/registry.h b/mooncake-store/include/io_pattern/registry.h index e560967310..518a2ef3b8 100644 --- a/mooncake-store/include/io_pattern/registry.h +++ b/mooncake-store/include/io_pattern/registry.h @@ -20,6 +20,7 @@ class OpsRegistry { public: using Factory = std::function()>; + // Registration is thread-safe; duplicate names are rejected. bool Register(std::string name, Factory factory) { if (name.empty() || !factory) { return false; diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index 2a39ae0cc7..1fb936e56d 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "tenant_id.h" @@ -66,7 +67,7 @@ struct InferenceMetrics { uint32_t layout_group{0}; uint32_t prefix_depth{0}; uint32_t prefix_fanout{0}; - uint32_t match_length{0}; + uint32_t match_length{0}; // tokens/blocks, as defined by the connector uint32_t continuous_prefix_length{0}; uint32_t token_count{0}; float recompute_cost{0.0F}; @@ -75,7 +76,7 @@ struct InferenceMetrics { struct AccessRecord { ObjectRef object; - uint64_t observed_at_ns{0}; + uint64_t observed_at_ns{0}; // monotonic nanoseconds uint64_t block_size{0}; uint64_t latency_us{0}; CacheTier tier{CacheTier::kL2Segment}; @@ -213,4 +214,34 @@ struct AdmissionResult { float confidence{0.0F}; }; +struct CacheViewEntry { + ObjectRef object; + CacheTier tier{CacheTier::kL2Segment}; + uint64_t bytes{0}; +}; + +struct CacheView { + uint64_t version{0}; + std::vector entries; +}; + +using KVMappingTable = std::vector; + +enum class CacheEventType : uint8_t { + kUnknown, + kInserted, + kRemoved, + kTierChanged, +}; + +struct CacheEvent { + CacheEventType type{CacheEventType::kUnknown}; + ObjectRef object; + CacheTier source_tier{CacheTier::kL2Segment}; + CacheTier target_tier{CacheTier::kL2Segment}; +}; + +using PolicyCommand = + std::variant; + } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/view_manager.h b/mooncake-store/include/io_pattern/view_manager.h new file mode 100644 index 0000000000..4ef636b71d --- /dev/null +++ b/mooncake-store/include/io_pattern/view_manager.h @@ -0,0 +1,17 @@ +#pragma once + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +// Owns the published cache view, not the storage operations that realize it. +class CacheViewManager { + public: + virtual ~CacheViewManager() = default; + + virtual CacheView ComputeView() const = 0; + virtual void PublishEvent(const CacheEvent& event) = 0; + virtual KVMappingTable GetGlobalMapping() const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern_analyzer.h b/mooncake-store/include/io_pattern_analyzer.h new file mode 100644 index 0000000000..7579ac39c1 --- /dev/null +++ b/mooncake-store/include/io_pattern_analyzer.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/analyzer.h" diff --git a/mooncake-store/include/io_pattern_collector.h b/mooncake-store/include/io_pattern_collector.h new file mode 100644 index 0000000000..448be4a72f --- /dev/null +++ b/mooncake-store/include/io_pattern_collector.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/collector.h" diff --git a/mooncake-store/include/io_pattern_registry.h b/mooncake-store/include/io_pattern_registry.h new file mode 100644 index 0000000000..9b08e9df83 --- /dev/null +++ b/mooncake-store/include/io_pattern_registry.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/registry.h" diff --git a/mooncake-store/include/io_pattern_types.h b/mooncake-store/include/io_pattern_types.h new file mode 100644 index 0000000000..a8a6fb2610 --- /dev/null +++ b/mooncake-store/include/io_pattern_types.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/types.h" diff --git a/mooncake-store/include/policy_engine.h b/mooncake-store/include/policy_engine.h new file mode 100644 index 0000000000..9861c9c2fe --- /dev/null +++ b/mooncake-store/include/policy_engine.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/policy_engine.h" diff --git a/mooncake-store/include/prefetch_ops.h b/mooncake-store/include/prefetch_ops.h new file mode 100644 index 0000000000..9ef1ade825 --- /dev/null +++ b/mooncake-store/include/prefetch_ops.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/ops.h" diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index cd74468a20..e35e2c6dd7 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include @@ -18,9 +20,78 @@ class TestEvictionOps final : public EvictionOps { } }; +class TestCollector final : public IoPatternCollector { + public: + void ReportInferenceMetrics(const InferenceMetrics& metrics) override { + inference_metrics = metrics; + } + void RecordAccess(const AccessRecord& record) override { + access_record = record; + } + void RecordStorageMetric(const StorageMetric& metric) override { + storage_metric = metric; + } + IoPatternSnapshot GetSnapshot() const override { return snapshot; } + + InferenceMetrics inference_metrics; + AccessRecord access_record; + StorageMetric storage_metric; + IoPatternSnapshot snapshot; +}; + +class TestAnalyzer final : public IoPatternAnalyzer { + public: + PatternResult Analyze(const IoPatternSnapshot&) const override { + return result; + } + WorkloadType DetectWorkloadType( + const IoPatternSnapshot&) const override { + return result.workload_type; + } + float CalculateConfidence(const ObjectRef&, + const IoPatternSnapshot&) const override { + return result.workload_confidence; + } + + PatternResult result{.workload_type = WorkloadType::kMixed, + .workload_confidence = 0.75F}; +}; + +class TestPrefetchOps final : public PrefetchOps { + public: + PrefetchPlan Evaluate(const PolicyContext&, + const TraceHistory&) const override { + return plan; + } + + PrefetchPlan plan; +}; + +class TestAdmissionOps final : public AdmissionOps { + public: + AdmissionResult Evaluate(const ObjectRef& object, CacheTier tier, + const PolicyContext&) const override { + return AdmissionResult{.object = object, + .target_tier = tier, + .decision = AdmissionDecision::kAdmit}; + } +}; + +class TestPrefetchExecutor final : public PrefetchExecutor { + public: + ErrorCode Execute(const PrefetchPlan& value) override { + plan = value; + return ErrorCode::OK; + } + + PrefetchPlan plan; +}; + TEST(IoPatternFrameworkTest, PublicSeamsRemainAbstract) { static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); @@ -37,6 +108,90 @@ TEST(IoPatternFrameworkTest, CacheTierMaskRepresentsAllTiers) { EXPECT_EQ(all_tiers, 0x0F); } +TEST(IoPatternFrameworkTest, MetricsKeepTenantAndLayoutIdentity) { + InferenceMetrics metrics; + metrics.object = {TenantId("tenant-a"), "prefix/block-1"}; + metrics.layout = CacheLayout::kHmaMultiGroup; + metrics.layout_group = 3; + metrics.match_length = 512; + + IoPatternSnapshot snapshot; + KeyMetrics key_metrics; + key_metrics.object = metrics.object; + key_metrics.match_length = metrics.match_length; + key_metrics.layout = metrics.layout; + key_metrics.layout_group = metrics.layout_group; + snapshot.keys.push_back(key_metrics); + + ASSERT_EQ(snapshot.keys.size(), 1); + EXPECT_EQ(snapshot.keys.front().object.tenant_id.value(), "tenant-a"); + EXPECT_EQ(snapshot.keys.front().object.key, "prefix/block-1"); + EXPECT_EQ(snapshot.keys.front().layout, CacheLayout::kHmaMultiGroup); + EXPECT_EQ(snapshot.keys.front().layout_group, 3); + EXPECT_EQ(snapshot.keys.front().match_length, 512); +} + +TEST(IoPatternFrameworkTest, CollectorAndAnalyzerExposeValueFlow) { + TestCollector collector; + collector.inference_metrics.object = + {TenantId("tenant-a"), "prefix/block-1"}; + collector.snapshot.generated_at_ns = 42; + collector.snapshot.keys.push_back( + KeyMetrics{.object = collector.inference_metrics.object}); + + collector.ReportInferenceMetrics(collector.inference_metrics); + AccessRecord access_record; + access_record.object = collector.inference_metrics.object; + access_record.observed_at_ns = 43; + access_record.is_hit = true; + collector.RecordAccess(access_record); + collector.RecordStorageMetric(StorageMetric{.source_id = "segment-1"}); + + const auto snapshot = collector.GetSnapshot(); + ASSERT_EQ(snapshot.keys.size(), 1); + EXPECT_EQ(collector.inference_metrics.object.key, "prefix/block-1"); + EXPECT_TRUE(collector.access_record.is_hit); + EXPECT_EQ(collector.storage_metric.source_id, "segment-1"); + + TestAnalyzer analyzer; + const auto result = analyzer.Analyze(snapshot); + EXPECT_EQ(result.workload_type, WorkloadType::kMixed); + EXPECT_FLOAT_EQ(analyzer.CalculateConfidence({}, snapshot), 0.75F); + EXPECT_EQ(analyzer.DetectWorkloadType(snapshot), WorkloadType::kMixed); +} + +TEST(IoPatternFrameworkTest, PolicyContextCarriesRawAndDerivedViews) { + PolicyContext context; + context.snapshot.generated_at_ns = 123; + context.analysis.workload_type = WorkloadType::kMixed; + context.analysis.workload_confidence = 0.75F; + + EXPECT_EQ(context.snapshot.generated_at_ns, 123); + EXPECT_EQ(context.analysis.workload_type, WorkloadType::kMixed); + EXPECT_FLOAT_EQ(context.analysis.workload_confidence, 0.75F); +} + +TEST(IoPatternFrameworkTest, PolicyCommandAndViewRemainValueTypes) { + const ObjectRef object{TenantId("tenant-b"), "block"}; + PrefetchCandidate candidate; + candidate.object = object; + candidate.bytes = 4096; + PrefetchPlan prefetch_plan; + prefetch_plan.strategy = PrefetchStrategy::kTimeout; + prefetch_plan.timeout_us = 1000; + prefetch_plan.candidates.push_back(candidate); + const PolicyCommand command = prefetch_plan; + ASSERT_TRUE(std::holds_alternative(command)); + EXPECT_EQ(std::get(command).candidates.front().object, + object); + + CacheView view; + view.version = 7; + view.entries.push_back({object, CacheTier::kL1Host, 4096}); + EXPECT_EQ(view.entries.front().tier, CacheTier::kL1Host); + EXPECT_EQ(view.entries.front().bytes, 4096); +} + TEST(IoPatternFrameworkTest, RegistryCreatesTypedOpsAndRejectsDuplicates) { OpsRegistry registry; @@ -55,5 +210,37 @@ TEST(IoPatternFrameworkTest, RegistryCreatesTypedOpsAndRejectsDuplicates) { EXPECT_EQ(registry.Create("missing"), nullptr); } +TEST(IoPatternFrameworkTest, ComposedEngineDelegatesAndDegradesSafely) { + auto eviction = std::make_shared(); + auto prefetch = std::make_shared(); + auto admission = std::make_shared(); + prefetch->plan.strategy = PrefetchStrategy::kWaitComplete; + ComposedPolicyEngine engine(eviction, prefetch, admission); + + const auto delegated = + engine.PlanEviction({}, CacheTier::kL1Host, 2048); + EXPECT_EQ(delegated.source_tier, CacheTier::kL1Host); + EXPECT_EQ(delegated.target_bytes, 2048); + + const auto prefetch_plan = engine.PlanPrefetch({}, {}); + EXPECT_EQ(prefetch_plan.strategy, PrefetchStrategy::kWaitComplete); + + const ObjectRef object{TenantId("tenant-a"), "key"}; + const auto admitted = engine.DecideAdmission( + object, CacheTier::kL2Segment, {}); + EXPECT_EQ(admitted.object, object); + EXPECT_EQ(admitted.target_tier, CacheTier::kL2Segment); + EXPECT_EQ(admitted.decision, AdmissionDecision::kAdmit); + + ComposedPolicyEngine degraded(nullptr, nullptr, nullptr); + const auto deferred = + degraded.DecideAdmission(object, CacheTier::kL2Segment, {}); + EXPECT_EQ(deferred.decision, AdmissionDecision::kDefer); + + TestPrefetchExecutor executor; + EXPECT_EQ(executor.Execute(prefetch_plan), ErrorCode::OK); + EXPECT_EQ(executor.plan.strategy, PrefetchStrategy::kWaitComplete); +} + } // namespace } // namespace mooncake::io_pattern From 97158471c80197b384f2571a8b8bfe6708bf1743 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 2 Sep 2026 10:07:05 +0800 Subject: [PATCH 3/4] =?UTF-8?q?io=20pattern=E4=BB=A3=E7=A0=81=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E6=AD=A5=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 1097 +++++++++++++++++ mooncake-store/include/cache_view_manager.h | 3 - mooncake-store/include/cfm_client_impl.h | 2 + mooncake-store/include/collector_impl.h | 2 + .../include/degrading_policy_engine.h | 2 + mooncake-store/include/feedback.h | 2 + .../include/io_pattern/cfm_channel.h | 20 + .../include/io_pattern/cfm_client_impl.h | 34 + .../include/io_pattern/cfm_ingress.h | 27 + .../include/io_pattern/cfm_protocol.h | 22 + mooncake-store/include/io_pattern/collector.h | 3 +- .../include/io_pattern/collector_impl.h | 64 + .../io_pattern/degrading_policy_engine.h | 43 + mooncake-store/include/io_pattern/feedback.h | 61 + .../include/io_pattern/io_pattern.h | 20 +- .../include/io_pattern/kmeans_analyzer.h | 28 + .../include/io_pattern/legacy_eviction_ops.h | 24 + .../include/io_pattern/observability.h | 39 + mooncake-store/include/io_pattern/ops.h | 4 + .../include/io_pattern/policy_engine.h | 281 +++++ .../include/io_pattern/policy_strategies.h | 74 ++ mooncake-store/include/io_pattern/reporter.h | 68 + .../include/io_pattern/resilient_analyzer.h | 40 + .../io_pattern/resilient_cfm_channel.h | 43 + .../include/io_pattern/rpc_transport.h | 126 ++ mooncake-store/include/io_pattern/runtime.h | 94 ++ .../io_pattern/sliding_window_analyzer.h | 50 + .../include/io_pattern/threshold_analyzer.h | 38 + .../include/io_pattern/tier_executor.h | 40 + mooncake-store/include/io_pattern/types.h | 28 + .../include/io_pattern/view_manager.h | 17 - mooncake-store/include/io_pattern_analyzer.h | 1 + mooncake-store/include/legacy_eviction_ops.h | 2 + mooncake-store/include/master_service.h | 9 + mooncake-store/include/observability.h | 2 + mooncake-store/include/policy_strategies.h | 3 + mooncake-store/include/reporter.h | 2 + mooncake-store/include/resilient_analyzer.h | 2 + .../include/resilient_cfm_channel.h | 2 + mooncake-store/include/rpc_transport.h | 2 + .../include/sliding_window_analyzer.h | 2 + mooncake-store/include/threshold_analyzer.h | 3 + mooncake-store/include/tier_executor.h | 2 + mooncake-store/src/CMakeLists.txt | 18 + .../src/io_pattern/cfm_client_impl.cpp | 31 + mooncake-store/src/io_pattern/cfm_ingress.cpp | 36 + .../src/io_pattern/cfm_protocol.cpp | 425 +++++++ .../src/io_pattern/collector_impl.cpp | 183 +++ .../io_pattern/degrading_policy_engine.cpp | 60 + mooncake-store/src/io_pattern/feedback.cpp | 58 + .../src/io_pattern/kmeans_analyzer.cpp | 161 +++ .../src/io_pattern/legacy_eviction_ops.cpp | 35 + .../src/io_pattern/observability.cpp | 63 + .../src/io_pattern/policy_strategies.cpp | 213 ++++ mooncake-store/src/io_pattern/reporter.cpp | 143 +++ .../src/io_pattern/resilient_analyzer.cpp | 67 + .../src/io_pattern/resilient_cfm_channel.cpp | 80 ++ .../src/io_pattern/rpc_transport.cpp | 111 ++ mooncake-store/src/io_pattern/runtime.cpp | 261 ++++ .../io_pattern/sliding_window_analyzer.cpp | 92 ++ .../src/io_pattern/threshold_analyzer.cpp | 134 ++ .../src/io_pattern/tier_executor.cpp | 35 + mooncake-store/src/master_service.cpp | 109 +- .../tests/io_pattern_framework_test.cpp | 957 +++++++++++++- mooncake-wheel/mooncake/io_pattern_bridge.py | 81 ++ .../mooncake/mooncake_connector_v1.py | 68 +- 66 files changed, 5822 insertions(+), 27 deletions(-) create mode 100644 docs/source/io_pattern_design.md delete mode 100644 mooncake-store/include/cache_view_manager.h create mode 100644 mooncake-store/include/cfm_client_impl.h create mode 100644 mooncake-store/include/collector_impl.h create mode 100644 mooncake-store/include/degrading_policy_engine.h create mode 100644 mooncake-store/include/feedback.h create mode 100644 mooncake-store/include/io_pattern/cfm_channel.h create mode 100644 mooncake-store/include/io_pattern/cfm_client_impl.h create mode 100644 mooncake-store/include/io_pattern/cfm_ingress.h create mode 100644 mooncake-store/include/io_pattern/cfm_protocol.h create mode 100644 mooncake-store/include/io_pattern/collector_impl.h create mode 100644 mooncake-store/include/io_pattern/degrading_policy_engine.h create mode 100644 mooncake-store/include/io_pattern/feedback.h create mode 100644 mooncake-store/include/io_pattern/kmeans_analyzer.h create mode 100644 mooncake-store/include/io_pattern/legacy_eviction_ops.h create mode 100644 mooncake-store/include/io_pattern/observability.h create mode 100644 mooncake-store/include/io_pattern/policy_strategies.h create mode 100644 mooncake-store/include/io_pattern/reporter.h create mode 100644 mooncake-store/include/io_pattern/resilient_analyzer.h create mode 100644 mooncake-store/include/io_pattern/resilient_cfm_channel.h create mode 100644 mooncake-store/include/io_pattern/rpc_transport.h create mode 100644 mooncake-store/include/io_pattern/runtime.h create mode 100644 mooncake-store/include/io_pattern/sliding_window_analyzer.h create mode 100644 mooncake-store/include/io_pattern/threshold_analyzer.h create mode 100644 mooncake-store/include/io_pattern/tier_executor.h delete mode 100644 mooncake-store/include/io_pattern/view_manager.h create mode 100644 mooncake-store/include/legacy_eviction_ops.h create mode 100644 mooncake-store/include/observability.h create mode 100644 mooncake-store/include/policy_strategies.h create mode 100644 mooncake-store/include/reporter.h create mode 100644 mooncake-store/include/resilient_analyzer.h create mode 100644 mooncake-store/include/resilient_cfm_channel.h create mode 100644 mooncake-store/include/rpc_transport.h create mode 100644 mooncake-store/include/sliding_window_analyzer.h create mode 100644 mooncake-store/include/threshold_analyzer.h create mode 100644 mooncake-store/include/tier_executor.h create mode 100644 mooncake-store/src/io_pattern/cfm_client_impl.cpp create mode 100644 mooncake-store/src/io_pattern/cfm_ingress.cpp create mode 100644 mooncake-store/src/io_pattern/cfm_protocol.cpp create mode 100644 mooncake-store/src/io_pattern/collector_impl.cpp create mode 100644 mooncake-store/src/io_pattern/degrading_policy_engine.cpp create mode 100644 mooncake-store/src/io_pattern/feedback.cpp create mode 100644 mooncake-store/src/io_pattern/kmeans_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/legacy_eviction_ops.cpp create mode 100644 mooncake-store/src/io_pattern/observability.cpp create mode 100644 mooncake-store/src/io_pattern/policy_strategies.cpp create mode 100644 mooncake-store/src/io_pattern/reporter.cpp create mode 100644 mooncake-store/src/io_pattern/resilient_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/resilient_cfm_channel.cpp create mode 100644 mooncake-store/src/io_pattern/rpc_transport.cpp create mode 100644 mooncake-store/src/io_pattern/runtime.cpp create mode 100644 mooncake-store/src/io_pattern/sliding_window_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/threshold_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/tier_executor.cpp create mode 100644 mooncake-wheel/mooncake/io_pattern_bridge.py diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md new file mode 100644 index 0000000000..1c9ab5818c --- /dev/null +++ b/docs/source/io_pattern_design.md @@ -0,0 +1,1097 @@ +--- +orphan: true +--- +## 5.4.1 设计概述 + +``` +flowchart TD + subgraph 推理框架层 ["推理框架层 (vLLM/SGLang)"] + MC["MooncakeConnector
(vLLM v1)"] + HC["HiCache Connector
(SGLang)"] + PC["Prefix Cache Manager"] + end + + subgraph CFM ["CFM (Cache Flow Manager)"] + COL["Collector
(数据采集)"] + ANA["Analyzer
(模式分析)"] + ENG["Policy Engine
(策略决策)"] + COL --> ANA --> ENG + EV["Eviction Ops"] + PF["Prefetch Ops"] + AD["Admission Ops"] + ENG --> EV & PF & AD + end + + MC -->|"CFM Client
(采集/策略/预取)"| COL + HC -->|"CFM Client"| COL + PC -->|"CFM Client"| COL + + subgraph CVM ["CVM (Cache View Manager)"] + VIEW["视图计算 / 发布 / 系统事件感知 / 全局 KV 映射表"] + end + + EV --> VIEW + PF --> VIEW + AD --> VIEW + + subgraph 存储层 ["存储层"] + L0["L0: HBM
(UB2PCIe/d2h)"] + L1["L1: Host DRAM/SSD
(计算节点本地内存/SSD(xds)"] + L2["L2: Segment DRAM
(池化内存 URMA mem)"] + L3["L3: Nof SSD
(SSU/远端池化 SSD)"] + end + + VIEW --> L0 + VIEW --> L2 + VIEW --> L3 + L0 <-.->|"tier down/up"| L1 + L1 <-.->|"tier down/up"| L2 + L2 <-.->|"offload/promotion"| L3 +``` + +**模块总体架构图** + +``` +classDiagram + class IoPatternCollector { + <> + +ReportInferenceMetrics(metrics) void + +RecordAccess(key, record) void + +RecordStorageMetric(metric) void + +GetSnapshot() IoPatternSnapshot + } + + class IoPatternAnalyzer { + <> + +AnalyzePattern(snapshot) PatternResult + +DetectWorkloadType(window) WorkloadType + +CalculateConfidence(key) float + } + + class PolicyEngine { + <> + +ExecutePolicy(context, tier, bytes, trace, admissions) PolicyResult + } + + class EvictionOps { + <> + +Evaluate(context, tier, bytes) EvictionPlan + } + + class PrefetchOps { + <> + +Evaluate(context, trace) PrefetchPlan + } + + class AdmissionOps { + <> + +Evaluate(object, tier, context) AdmissionResult + } + + class CfmClient { + +ReportInferenceMetrics(metrics) void + +ReceivePolicy指令() void + +ExecutePrefetch(candidates) void + } + + class ScoreBasedEviction { + +Evaluate(context, tier, bytes) EvictionPlan + } + + class TraceBasedPrefetch { + +Evaluate(context, trace) PrefetchPlan + } + + class PrefixMatchAdmission { + +Evaluate(object, tier, context) AdmissionResult + } + + PolicyEngine *-- EvictionOps : contains + PolicyEngine *-- PrefetchOps : contains + PolicyEngine *-- AdmissionOps : contains + IoPatternCollector --> IoPatternAnalyzer : reports + IoPatternAnalyzer --> PolicyEngine : analyzes + CfmClient --> IoPatternCollector : reports metrics + CfmClient --> PolicyEngine : receives policy + EvictionOps <|.. ScoreBasedEviction : implements + PrefetchOps <|.. TraceBasedPrefetch : implements + AdmissionOps <|.. PrefixMatchAdmission : implements +``` + +## 5.4.2 IO Pattern 三层架构 + +IO Pattern 模块采用**采集层 -> 分析层 -> 策略层**的三层架构。 + +### 5.4.2.1 采集层 (IO Pattern Collector) + +采集层负责从各数据源采集原始 IO 指标,采用异步上报机制避免阻塞数据路径。 + +**采集来源分三层:** + +| 采集层 | 数据源 | 采集指标 | 现有代码锚点 | +| ----------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| 推理框架层 | vLLM MooncakeConnector / SGLang HiCache Connector | prefix match length, request priority, token 序列, recompute cost | `mooncake_connector_v1.py`, SGLang hicache connector | +| SuperCache SDK 层 | Client / Master / SubMaster | Get/Put/Tier 命中率, 访问时序, key 频率, 前缀树深度/fanout, 副本分布, 迁移 ETA, 写路径指标 (batch_size, overwrite_ratio) | `client_service.cpp`, `master_service.cpp`, `local_hot_cache.cpp` | +| 存储后端层 | SSD/Nof Segment | 读写带宽, 读写延迟, GC 状态, 盘内 Superblock 布局, 容量水位 | `client_metric.h:SsdMetric`, `allocation_strategy.h:SsdMetricsProvider` | + +**采集机制设计:** + +1. **轻量级埋点**:复用和扩展现有 `CountMinSketch`(频率统计)、`SsdMetric`(SSD 延迟/吞吐)、`storage_backend.h:last_access_ns_`(最后访问时间)等埋点,避免重复建设 +2. **异步上报**:CFM Client 定期异步上报指标至 SubMaster,采用 batch 聚合减少 RPC 开销。上报间隔自适应负载(低负载 100ms,高负载退避至 500ms-1s) +3. **全局聚合**:SubMaster 聚合各节点上报的指标,维护全局 token 指标流动视图 +4. **采样降级**:在高负载场景下支持采样率动态调整,优先保障数据路径性能 +5. **多租户隔离**:指标按 `TenantId` 分桶采集,避免高频租户淹没低频租户,沿用现有 `CountMinSketch` 的 `tenant_id.MakeScopedKey` 模式 + +固定 100ms 上报间隔在大规模集群下可能产生可观开销。采用自适应间隔: + +| 负载状态 | 上报间隔 | 触发条件 | +| -------- | -------- | ----------------------------------------- | +| 低负载 | 100ms | mem_used_ratio < 50% | +| 中负载 | 200ms | 50% <= mem_used_ratio < 80% | +| 高负载 | 500ms | mem_used_ratio >= 80% | +| 极高负载 | 1000ms | mem_used_ratio >= 95% 或 RPC 延迟 > 100ms | + +> +> 高负载时拉长间隔减少 RPC 开销,但保持最低 1s 上报频率确保策略时效性。量化估算:4000 节点集群,100ms 间隔下每秒 40000 RPC,单 RPC ~2KB,总带宽 ~80MB/s + +#### 5.4.2.1.1 指标采集与上报流程 + +``` +sequenceDiagram + participant INF as 推理框架 (vLLM/SGLang) + participant CFM as CFM Client + participant SUB as SubMaster + + INF->>CFM: 1. 请求完成/前缀匹配 + CFM->>SUB: 2. 批量上报指标 (InferenceMetrics) + CFM->>SUB: 3. SDK 层埋点 (AccessRecord) + CFM->>SUB: 4. 存储后端指标 (StorageMetric) + Note over SUB: 5. 全局聚合
IoPatternSnapshot +``` + +#### 5.4.2.1.2 指标分类 + +IO Pattern 采集指标分为六大类,对应分级缓存淘汰/准入/预取流程的采集指标定义: + +**时序指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| --------------------- | --------- | ------------------------ | ---------- | ----------------------------------- | +| `last_access_time` | timestamp | 最近一次访问时间 | SDK 层 | `storage_backend.h:last_access_ns_` | +| `access_count_window` | uint32 | 最近时间窗口内访问次数 | SDK 层 | `count_min_sketch.h:CountMinSketch` | +| `idle_time` | duration | = now - last_access_time | 分析层计算 | - | + +**价值指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| ---------------- | ------ | ------------------------------ | ---------- | ---------------- | +| `recompute_cost` | float | 重新计算时间 (token 数 / 时间) | 推理框架层 | connector 层估算 | +| `block_size` | uint64 | 数据大小 (bytes) | SDK 层 | object metadata | +| `token_count` | uint32 | token 数 | 推理框架层 | connector 层 | + +**结构指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| -------------------------- | ------ | ---------------------------- | ---------- | -------------------- | +| `prefix_depth` | uint32 | 前缀深度 (prefix tree level) | 推理框架层 | prefix cache manager | +| `prefix_fanout` | uint32 | 共享该前缀的请求/分支数量 | 推理框架层 | prefix cache manager | +| `match_length` | uint32 | 前缀匹配长度 | 推理框架层 | connector 层 | +| `continuous_prefix_length` | uint32 | 连续前缀长度 | 推理框架层 | connector 层 | + +**副本指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| --------------------- | -------- | -------------------------- | ---------- | -------------------------- | +| `replica_tiers` | bitmap | 当前在哪些层有副本 (L0-L3) | SDK 层 | `master_service.h:Replica` | +| `transfer_eta` | duration | 迁移路径预计耗时 | 分析层计算 | - | +| `ssd_replica_exists` | bool | SSD 层是否有副本 | SDK 层 | replica metadata | +| `other_replica_count` | uint32 | 其他层副本数 | SDK 层 | replica metadata | + +**状态指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| -------- | ---- | ------------ | -------- | ----------------------------- | +| `active` | bool | 是否正在使用 | SDK 层 | `local_hot_cache.h:ref_count` | +| `pinned` | bool | 是否不可迁移 | SDK 层 | promotion task pinning | + +**存储后端指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| ------------------- | --------- | ----------------- | ---------- | ---------------------------------------------------------------------------- | +| `ssd_read_latency` | histogram | SSD 读延迟分布 | 存储后端层 | `client_metric.h:SsdMetric` | +| `ssd_write_latency` | histogram | SSD 写延迟分布 | 存储后端层 | `client_metric.h:SsdMetric` | +| `ssd_gc_status` | enum | SSD GC 状态 | 存储后端层 | Nof TGT | +| `mem_used_ratio` | float | DRAM 内存水位比例 | SDK 层 | `MasterMetricManager::get_global_mem_used_ratio()` (Master 侧全局 DRAM 水位) | +| `ssd_used_bytes` | int64 | SSD 已用容量 | 存储后端层 | `allocation_strategy.h:SsdMetricsProvider` | + +#### 5.4.2.1.3 指标采集接口 + +``` +// IO Pattern Collector 接口 (新增, 位于 include/io_pattern_collector.h) +class IoPatternCollector { + public: + virtual ~IoPatternCollector() = default; + + // 推理框架层指标 (通过 CFM Client 上报) + virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; + + // SDK 层指标 (内部埋点) + virtual void RecordAccess(const std::string& key, + const AccessRecord& record) = 0; + + // 存储后端层指标 + virtual void RecordStorageMetric(const StorageMetric& metric) = 0; + + // 获取聚合后的指标快照 + virtual IoPatternSnapshot GetSnapshot() const = 0; +}; + +struct AccessRecord { + std::string key; + std::chrono::steady_clock::time_point access_time; + uint64_t block_size; + ReplicaType replica_type; + bool is_hit; + std::chrono::microseconds latency; +}; + +struct InferenceMetrics { + std::string session_id; + uint32_t prefix_depth; + uint32_t prefix_fanout; + uint32_t match_length; + uint32_t continuous_prefix_length; + uint32_t token_count; + float recompute_cost; + uint8_t request_priority; +}; +``` + +### 5.4.2.2 分析层 (IO Pattern Analyzer) + +分析层对采集的原始指标进行模式识别和特征提取,输出结构化的 IO Pattern 描述。 + +**分析能力:** + +| 分析类型 | 描述 | 输入指标 | 输出 | 对应需求 | +| ------------- | ------------------------------ | ------------------------------------------------------ | ------------------------------------ | -------------- | +| 热度分析 | 基于 LFU/滑动窗口的频率统计 | access_count_window, last_access_time | hot/cold 分类, 频率评分 | 淘汰/准入/预取 | +| 前缀分析 | Prefix tree 深度和 fanout 分析 | prefix_depth, prefix_fanout, match_length | 前缀共享度, 预取候选 | 预取/准入 | +| 时序预测 | 访问间隔和空闲时间分析 | idle_time, access pattern, access_count_window | 空闲评分, 预取优先级 | 淘汰/Tier down | +| 代价评估 | 重计算代价和迁移代价评估 | recompute_cost, token 数, block_size, transfer_eta | 代价评分, 迁移 ROI | 淘汰/Tier up | +| 访问模式识别 | 顺序/随机、大包/小包、读写比 | IO size distribution, access sequence | 模式分类 (SEQ/RANDOM/KV_LOOKUP等) | 缓存分区/分流 | +| 副本分析 | 多层副本分布分析 | replica_tiers, active/pinned | 副本冗余度, 迁移安全性 | 淘汰/Tier down | +| 写路径分析 | 写入模式分析 | write_batch_size, write_burst, overwrite_ratio | 写穿风险, GC 预警 | 准入/GC | +| Workload 识别 | 推理场景类型自动识别 | token_count, prefix_fanout, block_size, frequency 分布 | workload_type (Code Agent/推荐/对话) | 策略模板选择 | + +**Workload Type 感知策略模板** + +不同推理场景的 IO Pattern 差异巨大,单一通用评分公式无法覆盖所有场景。 + +**配置参数:** + +| 参数 | 默认值 | 说明 | +| -------------------------------------- | ------------- | ------------------------------------------------------------------ | +| `workload_detection_window_sec` | 60 | workload 识别滑动窗口大小 | +| `workload_detection_method` | `auto` | 识别方法:`auto`(阈值+聚类)、`threshold`(仅阈值)、`kmeans`(仅聚类) | +| `workload_template_transition_windows` | 3 | 模板切换过渡窗口数,控制平滑度 | +| `workload_mixed_load_mode` | `per_session` | 混合负载处理:`per_session`(按会话标记)、`global`(全局统一) | + +**运维观测:** + +| 指标 | 描述 | +| --------------------------------------- | ------------------------ | +| `workload_current_type` | 当前识别的 workload type | +| `workload_type_switch_count` | workload type 切换次数 | +| `workload_detection_latency_us` | 单次识别延迟 | +| `workload_template_transition_progress` | 模板过渡进度 (0.0-1.0) | + +**Workload 识别机制:** + +不同推理场景的 KVCache 访问模式差异巨大,单一通用评分公式无法覆盖所有场景。IO Pattern 分析层基于滑动窗口内的指标分布统计,自动识别 workload type 并切换对应策略模板。 + +**Workload Type 特征矩阵:** + +| Workload Type | 典型场景 | 访问特征 | 识别信号 | +| ------------- | ------------------------------ | ------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Code Agent | Cursor/Copilot 长程代码生成 | 长上下文(>32K token)、高前缀复用、大 block_size(>512KB)、多轮访问为主 | 高 token_count + 低 prefix_fanout + 大 block_size + 低 frequency | +| 生成式推荐 | 京东/字节 GR 精排召回 | 高频小 block(<128KB)、高复用、密集访问、低重计算代价 | 低 token_count + 高 frequency + 小 block_size + 低 recompute_cost | +| 多轮对话 | ChatGPT 类对话、Agent 工具调用 | 中等 block、高前缀共享、渐进式增长、prefix cache 命中率高 | 中 token_count + 高 prefix_fanout + 高 match_length + 高 recompute_cost | + +**识别算法:** + +分析层维护一个滑动窗口(默认 60s),统计窗口内所有请求的 `token_count`/`prefix_fanout`/`block_size`/`frequency` 四维分布。采用两阶段识别: + +``` +阶段 1: 特征提取 + for each request in window: + feature_vector = (median(token_count), p90(prefix_fanout), + median(block_size), median(frequency)) + +阶段 2: 分类决策 + if feature_vector matches阈值规则: + -> 直接分类 (快速路径, 延迟 < 1ms) + else: + -> K-means 聚类 (慢速路径, 延迟 < 10ms, 用于混合负载场景) +``` + +**阈值规则(快速路径):** + +| 判定条件 | → Workload Type | +| ---------------------------------------------------------------------------------- | ----------------------- | +| `median(token_count) > 16KB && p90(prefix_fanout) > 16 && p90(match_length) > 256` | Code Agent | +| `median(block_size) < 128KB && median(frequency) > 20` | 生成式推荐 | +| `p90(prefix_fanout) > 16 && p90(match_length) > 256` | 多轮对话 | +| 不满足以上任一 | 混合负载 → K-means 聚类 | + +**策略模板对照:** + +每种 workload type 对应一组完整的策略参数模板,覆盖淘汰/准入/预取/Tier 四个维度: + +| 策略维度 | Code Agent | 生成式推荐 | 多轮对话 | +| ------------- | ----------------------------------------- | --------------------------------------- | -------------------------------------- | +| **预取** | 保守(仅 prefix > 512 预取,best_effort) | 激进(prefix > 64 预取,wait_complete) | 前缀优先(prefix > 256 预取,timeout) | +| **淘汰** | 激进(低 idle_thres,快速释放 L0) | 保守(高 idle_thres,保留热数据) | 前缀感知(prefix_fanout 高权重保留) | +| **准入** | 低阈值(access_count > 2 即准入) | 高阈值(access_count > 20 才准入) | 前缀准入(match_length > 128 即准入) | +| **Tier down** | 快速降级(L0→L2 跳级,跳过 L1) | 缓慢降级(L0→L1→L2 逐层) | 前缀亲和(共享前缀的 block 同层迁移) | +| **淘汰权重** | α=0.8, γ=0.2, δ=0.3, ε=0.1 | α=0.3, γ=0.8, δ=0.2, ε=0.1 | α=0.5, γ=0.4, δ=0.6, ε=0.8 | + +**模板切换机制:** + +``` +flowchart TD + WIN["滑动窗口指标统计
(60s)"] + FEAT["特征提取
4 维分布向量"] + RULE["阈值规则匹配"] + KMEANS["K-means 聚类
(混合负载)"] + CLASSIFY["Workload Type 判定"] + TEMPLATE["策略模板加载
(淘汰/准入/预取/Tier 参数)"] + APPLY["应用至 Policy Engine"] + + WIN --> FEAT --> RULE + RULE -->|"匹配成功"| CLASSIFY + RULE -->|"不匹配"| KMEANS --> CLASSIFY + CLASSIFY --> TEMPLATE --> APPLY +``` + +**切换平滑性:** workload type 变化时,策略参数不是瞬间切换,而是通过加权过渡(新旧模板权重在 3 个窗口周期内从 100:0 渐变到 0:100),避免策略突变导致缓存抖动。 + +**混合负载处理:** 当 K-means 识别出多种 workload type 共存时(如同一集群同时服务对话和推荐),采用 per-session workload 标记——在请求入口处根据 session 特征打标签,各 session 独立使用对应模板,而非全局统一。 + +### 5.4.2.3 策略层 (IO Pattern Policy Engine) + +策略层基于分析层的输出,通过可注册的 Ops 接口驱动各缓存机制: + +``` +flowchart TD + PE["Policy Engine"] + + subgraph EvictionOps ["EvictionOps (淘汰策略)"] + LRU["LRU"] + LFU["LFU"] + SBE["ScoreBased
(L0-L3 四层)"] + end + + subgraph PrefetchOps ["PrefetchOps (预取策略)"] + BE["BestEffort"] + TO["Timeout"] + WC["WaitComplete"] + TB["TraceBased"] + end + + subgraph AdmissionOps ["AdmissionOps (准入策略)"] + FREQ["Frequency"] + PM["PrefixMatch"] + WM["Watermark"] + CA["CostAware"] + end + + PE --> EvictionOps + PE --> PrefetchOps + PE --> AdmissionOps +``` + +## 5.4.3 缓存机制集成与关键流程 + +IO Pattern 模块不是重写现有机制,而是在现有机制之上增加统一的数据采集和分析层,通过 Ops 抽象接口驱动各机制。 + +### 5.4.3.2 淘汰 (Eviction) + +**现有机制**:`EvictionStrategy` 抽象类(`eviction_strategy.h`)提供 LRU 和 FIFO 两种实现;`storage_backend.h` 中基于 `last_access_ns_` 维护 LRU 索引。 + +**IO Pattern 增强**:引入基于评分的淘汰策略 (ScoreBasedEviction),按4层缓存层级分别使用不同评分公式。所有指标先经归一化处理(`normalize(x) = x / max_observed_x`,映射到 `[0, 1]`),消除量纲差异后再加权求和。归一化基准基于滑动窗口(默认 60s)内的最大观测值动态更新。 + +``` +flowchart TD + SNAP["IO Pattern Snapshot"] + NORM["归一化处理
norm(x) = x / max_observed"] + PE["Policy Engine
选择层级策略"] + SNAP --> NORM --> PE + + PE --> L0S["L0 HBM Evict"] + PE --> L1S["L1 Host DRAM/SSD Evict"] + PE --> L2S["L2 Segment DRAM Evict"] + PE --> L3S["L3 Nof SSD Evict"] + + L0S --> L0F["α\*norm(idle) - γ\*norm(freq)
- δ\*norm(recompute) - ε\*norm(fanout)"] + L1S --> L1F["α\*norm(idle) - γ\*norm(freq)
+ δ\*norm(lower_replica)
- ε\*norm(fanout) - ζ*norm(recompute)"] + L2S --> L2F["α\*norm(idle) - γ\*norm(freq)
+ δ\*norm(lower_replica)
- ε\*norm(fanout) - ζ*norm(recompute)"] + L3S --> L3F["α\*norm(idle)\*norm(block_size)
- γ\*norm(freq) - δ\*norm(recompute)
+ η*norm(other_replica)"] + + L0F --> L0OUT["-> L1"] + L1F --> L1OUT["-> L2"] + L2F --> L2OUT["-> L3"] + L3F --> L3OUT["-> 丢弃"] +``` + +**各层淘汰策略说明:** + +| 层级 | 评分侧重 | 淘汰去向 | 说明 | +| ---------------- | --------------------------------------------------------------------- | -------- | ------------------------- | +| L0 HBM | `idle_time` 主导,`recompute_cost`/`prefix_fanout` 高权重保留 | → L1 | HBM 最贵,冷数据快速降级 | +| L1 Host DRAM/SSD | 下层已有副本可安全淘汰,`prefix_fanout`/`recompute_cost` 高的数据保留 | → L2 | 本地 DRAM/SSD 到远端 DRAM | +| L2 Segment DRAM | `prefix_fanout`/`recompute_cost` 高的数据保留 | → L3 | 池化内存到远端 SSD | +| L3 Nof SSD | `block_size` 大 + 其他层已有副本优先淘汰 | → 丢弃 | 最底层,无下降空间 | + +**集成方式**:扩展现有 `EvictionStrategy` 接口,新增 `ScoreBasedEvictionStrategy`,由 Policy Engine 根据层级动态选择策略。 + +### 5.4.3.3 准入 (Admission) + +**现有机制**: + +- Client 侧:`CountMinSketch` + `admission_threshold_` 频率准入(`client_service.cpp`),仅频繁访问的 key 提升 hot cache +- Master 侧:Promotion-on-Hit 的 `promotion_admission_threshold_` 频率门控 + watermark 门控(`master_service.cpp`) + +**IO Pattern 增强**:扩展准入策略为多层逐级准入控制,每层提升需满足对应条件: + +| 准入路径 | 条件 | 说明 | +| -------- | -------------------------------------------------------------- | ----------------------------------------------------- | +| L3→L2 | `access_count_window >= threshold` | 频率达标才从 Nof SSD 提升至 Segment DRAM | +| L2→L1 | `access_count_window >= threshold && upper_space <= max_space` | 频率达标且上层有空间才提升 Segment DRAM→Host DRAM/SSD | +| L1→L0 | `max_length >= threshold (64)` | 前缀长度达标才从 Host DRAM/SSD 提升至 HBM | + +> +> SSD→HBM 跨层直达(跳过中间层)仅由推理框架 prefix cache 命中时触发,Mooncake 侧不自主执行跨层晋升到 HBM。 + +``` +flowchart TD + REQ["访问请求"] + ANA["IO Pattern Analyzer
计算准入条件"] + REQ --> ANA + + ANA --> P1["L3→L2
access_cnt >= thres"] + ANA --> P2["L2→L1
access_cnt >= thres
&& upper_space <= max"] + ANA --> P3["L1→L0
max_length >= 64"] + + P1 --> FA["频率准入
(CountMin Sketch)"] + P2 --> FA2["频率+空间准入
(Frequency + Watermark)"] + P3 --> PA["前缀准入
(PrefixMatch Admission)"] +``` + +**集成方式**:扩展现有 `CountMinSketch` 准入逻辑,新增 `PrefixMatchAdmission` 和 `CostAwareAdmission` 策略。 + +### 5.4.3.4 Tier Down / SSD Offload + +**现有机制**: + +- `enable_ssd_offload` + `ssd_offload_path` 配置 SSD offload 路径(`real_client.cpp`) +- `offload_on_evict` 模式:在淘汰时延迟 offload 到 LOCAL_DISK(`master_service.cpp`) +- `offload_force_evict`:超过 offload cap 时直接淘汰不 offload + +**IO Pattern 增强**:基于热度阈值的逐级 tier down,数据按 L0→L1→L2→L3 顺序逐层降级: + +``` +flowchart TD + START["L0 HBM 容量/水位检测"] + C1{"idle_time >= L0 cold_thres || frequency < L0 hot_thres ?"} + C2{"L2 seg_dram_avail ?"} + C3{"L3 nof SSD avail ?"} + C4{"L1 host DRAM<= thres ?"} + C5{"xds available ?"} + C6{"idle_time >= L1 cold_thres || frequency < L1 hot_thres ?"} + C7{"idle_time >= L2 cold_thres || frequency < L2 hot_thres ?"} + TD_L1A["L1 Host DRAM"] + TD_L1B["L1 SSD(xds)"] + TD_L2["L2 Segment DRAM"] + TD_L3["L3 Nof SSD"] + + START --> C1 + C1 -->|是| C4 + C4 -->|是| TD_L1A + C4 -->|否| C5 + C5 -->|是| TD_L1B + C5 -->|否| C2 + C2 -->|是| TD_L2 + C2 -->|否| C3 + C3 -->|否| WAIT["下层均不可用
等待重试 / 强制 evict"] + C3 -->|是| TD_L3 + TD_L1A --> C6 + C6 -->|是| C2 + TD_L2 --> C7 + C7 -->|是| C3 +``` + +> +> 当所有下层均不可用时,数据暂留当前层并等待下层恢复,或触发强制 evict 释放空间。冷数据不会保留在高速层——高速层是最昂贵的资源,冷数据必须逐级降级。 + +### 5.4.3.5 Tier Up / Promotion-on-Hit + +**现有机制**: + +- `promotion_on_hit` 模式(`master_service.cpp:379`):Get 观察到 LOCAL_DISK-only key 时队列异步拷贝回 MEMORY +- `CountMinSketch` 频率门控(`master_service.cpp:6944`) +- watermark 门控:DRAM 低于 `eviction_high_watermark_ratio_` 才允许 promotion +- `promotion_queue_limit` + `promotion_max_per_heartbeat` 控制 promotion 速率 +- `PromotionCandidate` 跟踪 + 重试 + TTL 过期 + +**IO Pattern 增强**:IO Pattern 分析层为 promotion 提供更丰富的决策输入: + +- 前缀匹配度:高前缀匹配的 key 优先 promotion +- 重计算代价:高 recompute_cost 的 key 优先 promotion +- 迁移 ETA:根据带宽和 block_size 估算 transfer_eta,避免迁移耗时过长 + +> +> Mooncake 侧 promotion 仅执行逐级提升(L3→L2→L1),不自主晋升到 L0 HBM。L0 HBM 层的数据加载由推理框架 prefix cache 命中时自主触发。 + +``` +flowchart TD + START["prefix cache 命中/预取触发/get"] + CALC["Tier Up Priority 计算:
priority = w0 * recompute_cost
+ w1 * continuous_prefix
+ w2 * request_priority
- w3 * transfer_eta"] + SORT["按优先级排序"] + EXEC["执行逐级 tier up
L3→L2→L1"] + + START --> CALC --> SORT --> EXEC +``` + +### 5.4.3.6 预取 (Prefetch) + +**现有机制**:当前无显式预取机制,Promotion-on-Hit 在 Get 命中 LOCAL_DISK 时异步提升到 MEMORY。 + +**IO Pattern 增强**:新增 `PrefetchOps` 抽象,SubMaster 根据 trace 和置信阈值生成预取器: + +**预取触发条件:** 低速层 prefix match length > 阈值 (256) 时触发预取至上一层(如 L3→L2、L2→L1) + +**预取策略(三种模式):** + +| 策略 | 描述 | 适用场景 | +| --------------- | ----------------------------------- | -------------------- | +| `best_effort` | check & match,无论是否完成立即返回 | 对 TTFT 时延敏感业务 | +| `timeout` | 预取完成或超时立即返回 | 兼顾时延和命中率 | +| `wait_complete` | 死等数据加载完成 | 追求极致命中率 | + +分析层基于 trace 历史命中率和置信阈值生成策略输入。置信度 = 滑动窗口内命中次数 / 总访问次数,低于阈值时不触发操作避免误判。例如预取器生成: + +- SubMaster 根据 trace 历史 + 置信阈值(如 `confidence > 0.6 && prefix match length > 256`)生成预取器 +- 置信度低于阈值时不触发操作,避免误判导致的缓存污染 +- 置信阈值精确定义和各策略默认值详见 + +**置信度计算:** 基于滑动窗口内的历史命中率,衡量当前预测的可信程度。 + +``` +confidence = hit_count_in_window / total_access_in_window +``` + +**置信阈值应用:** + +| 策略 | 置信阈值 | 含义 | 默认值 | +| -------- | ----------------------------------------------- | --------------------------------------- | ------------------------------------ | +| 预取触发 | `confidence > 0.6 && match_length > 256` | 历史命中率 > 60% 且前缀匹配足够长才预取 | prefix_threshold=256, confidence=0.6 | +| 准入提升 | `confidence > 0.5 && access_count >= threshold` | 历史命中率 > 50% 且频率达标才提升 | confidence=0.5 | +| 淘汰保守 | `confidence > 0.8` 时降低淘汰权重 | 高置信热数据更保守淘汰 | confidence=0.8 | + +> +> 置信度低于阈值时不触发操作,避免误判导致的缓存污染。置信度窗口默认 60s,可通过 `confidence_window_sec` 配置。 + +预取流程仅看 `match_length > 256` 触发 + +``` +flowchart TD + START["低速层 prefix match
(L1-L3)"] + C1{"match_length > 256 ?"} + NOP["不预取"] + SEL["选择预取策略"] + + START --> C1 + C1 -->|否| NOP + C1 -->|是| SEL + SEL --> BE["best_effort"] + SEL --> TO["timeout"] + SEL --> WC["wait_complete"] +``` + +- `max_prefetch_ratio`:预取占用带宽上限比例,默认 20%,可通过 `prefetch_max_bw_ratio` 配置 +- 带宽不足时延迟重试,而非直接丢弃预取请求 + +**集成方式**:在 SubMaster 中新增预取器,根据 IO Pattern 分析层的置信阈值异步预取 key 至上层。 + +## 5.4.5 上层推理框架对接 + +### 5.4.5.1 vLLM 集成 + +**现有对接**:`MooncakeConnector`(`mooncake_connector_v1.py`)实现 vLLM `KVConnectorBase_V1` 接口,支持 PD disaggregation(Prefill/Decode 分离)。 + +> +> **上层框架改动**:vLLM 侧无需改动。`MooncakeConnector` 作为 vLLM 的 out-of-tree connector(通过 `--kv_connector_module_path` 加载),在 connector 内部新增 CFM Client 调用即可上报指标和接收策略指令,不涉及 vLLM scheduler/engine 接口变更。vLLM v0.13.0+ 已内置 mooncake connector,后续可考虑将 CFM Client 合入上游。 + +**IO Pattern 对接增强**: + +``` +flowchart TD + VLLM["vLLM Engine"] + + subgraph MC ["MooncakeConnector (KVConnectorBase_V1)"] + GNMT["get_num_new_matched_tokens()
上报 match_length"] + USA["update_state_after_alloc()
上报 prefix_depth"] + RF["request_finished()
上报 token_count, recompute_cost"] + end + + subgraph CFMC ["CFM Client (新增)"] + RIM["IoPatternCollector
.ReportInferenceMetrics()"] + POP["PrefetchOps
接收预取指令"] + AOP["AdmissionOps
接收准入策略"] + end + + VLLM --> MC + VLLM --> CFMC + GNMT --> RIM + USA --> RIM + RF --> RIM +``` + +**采集对接**:在 `MooncakeConnector` 中增加 CFM Client 调用,将以下指标上报至 IO Pattern Collector: + +- `match_length`:prefix cache 命中长度(来自 `get_num_new_matched_tokens()`) +- `prefix_depth` / `prefix_fanout`:前缀树结构(来自 `update_state_after_alloc()` 及 prefix cache manager) +- `token_count`:请求 token 数(来自 `request_finished()`) +- `recompute_cost`:重计算代价估算(connector 侧基于 token_count 和模型 FLOPS 估算) +- `request_priority`:请求优先级(connector 层从 request metadata 提取) + +**策略对接**:CFM Client 接收 Policy Engine 的策略指令: + +- 预取指令:根据 prefix match length > 256 触发异步预取 +- 准入指令:根据频率/前缀匹配控制数据提升层级 +- 淘汰指令:根据淘汰评分驱动 L0-L3 层间淘汰 + +### 5.4.5.2 SGLang 集成 + +**现有对接**:SGLang HiCache 通过 `--hicache-storage-backend: mooncake` 将 Mooncake 作为存储后端,支持 layer_first / page_first 布局。 + +**IO Pattern 对接增强**: + +``` +flowchart TD + SGL["SGLang Engine"] + + subgraph HCC ["HiCache Connector"] + HR["hicache-ratio
容量配比"] + HML["hicache-mem-layout
layer_first / page_first"] + HIO["hicache-io-backend
direct / async"] + end + + subgraph CFMS ["CFM Client (新增)"] + RIM2["IoPatternCollector
.ReportInferenceMetrics()"] + DLA["数据布局适配
(layer_first / page_first)"] + PAS["预取/准入/淘汰策略"] + end + + SGL --> HCC + SGL --> CFMS + HR --> RIM2 + HML --> DLA + HIO --> RIM2 +``` + +**数据布局适配**:IO Pattern 需感知推理框架的 KV cache 布局模式,vLLM 和 SGLang 均需适配: + +| 框架 | 布局模式 | 描述 | IO Pattern 适配 | +| ------ | ------------------------------ | ------------------------------------------------------------------------- | ----------------------------------------------- | +| SGLang | `layer_first` | (2, layer, slot, num_head, head_dim) | 前缀分析按 layer 维度,预取按 layer 批量 | +| SGLang | `page_first` | (2, page_num, layer, num_head, head_dim) | 前缀分析按 page 维度,预取按 page 批量 | +| SGLang | `page_first_direct` | 混合模型 (Full Attention + SWA/Mamba) | 分区准入,full KV 固定分区 + SWA/mamba 灵活分配 | +| vLLM | `page-based` (block_size 粒度) | vLLM v1 默认 page-based 布局,connector 通过 `get_kv_cache_layout()` 检测 | 前缀分析按 block 维度,预取按 block 批量 | +| vLLM | `HMA multi-group` | 混合模型 (attention + Mamba2),`SupportsHMA` 多 group 布局 | 分组准入,各 group 独立淘汰/预取策略 | + +> +> vLLM connector 已在初始化时调用 `get_kv_cache_layout()` 检测布局(`mooncake_connector_v1.py:511`),并通过 `SupportsHMA` 支持 hybrid 模型多 group 布局。SGLang 通过 `--hicache-mem-layout` 参数显式配置布局。两者均需在 CFM Client 上报时附带布局信息,供 IO Pattern 分析层选择对应的预取/准入粒度。 + +### 5.4.5.3 CFM Client 设计 + +CFM Client 部署在推理节点侧,作为推理框架与 SuperCache 之间的策略桥梁: + +``` +flowchart TD + subgraph CFMClient ["CFM Client"] + MR["Metrics Reporter
(采集上报)"] + PR["Policy Receiver
(策略接收)"] + PE["Prefetch Executor
(预取执行)"] + RPC["CFM RPC Channel
(to SubMaster / PrefixCache Master)"] + MR --> RPC + PR --> RPC + PE --> RPC + end +``` + +**职责:** + +1. **Metrics Reporter**:定期(100ms)批量上报推理框架指标至 SubMaster +2. **Policy Receiver**:接收 Policy Engine 的淘汰/预取/准入策略指令 +3. **Prefetch Executor**:执行异步预取,支持 best_effort / timeout / wait_complete 三种模式 + +## 5.4.6 Ops 抽象接口设计 + +> **接口修订(2026-09)**:本节原始的 string/vector 简化签名仅用于 +> 查询示例,不能承载租户、字节预算、评分、Tier、超时和置信度等执行 +> 元数据。实际实现统一采用文末“Revised Ops contract”中的完整计划接口。 + +### 5.4.6.1 EvictionOps + +``` +// include/eviction_ops.h (扩展现有 eviction_strategy.h) +class EvictionOps { + public: + virtual ~EvictionOps() = default; + + // 基于 IO Pattern 评分选择淘汰 key + virtual std::vector SelectEvictionCandidates( + const IoPatternSnapshot& snapshot, + CacheTier tier, + size_t target_bytes) = 0; + + // 注册淘汰算法 + static void Register(const std::string& name, + std::function()> factory); +}; + +// 已有实现: LRU, FIFO (eviction_strategy.h) +// 新增实现: ScoreBasedEviction (L0-L3 四层不同评分公式) +``` + +### 5.4.6.2 PrefetchOps + +``` +// include/prefetch_ops.h (新增) +class PrefetchOps { + public: + virtual ~PrefetchOps() = default; + + // 基于 trace 和置信阈值生成预取候选 + virtual std::vector GeneratePrefetchPlan( + const IoPatternSnapshot& snapshot, + const TraceHistory& trace) = 0; + + // 执行预取 + virtual ErrorCode ExecutePrefetch( + const std::vector& candidates, + PrefetchStrategy strategy) = 0; + + static void Register(const std::string& name, + std::function()> factory); +}; + +enum class PrefetchStrategy { + kBestEffort, // 无论是否完成立即返回 + kTimeout, // 预取完成或超时立即返回 + kWaitComplete, // 死等数据加载完成 +}; +``` + +### 5.4.6.3 AdmissionOps + +``` +// include/admission_ops.h (新增, 扩展现有 CountMinSketch 准入) +class AdmissionOps { + public: + virtual ~AdmissionOps() = default; + + // 准入决策:是否允许数据进入目标层 + virtual AdmissionDecision CheckAdmission( + const std::string& key, + CacheTier target_tier, + const IoPatternSnapshot& snapshot) = 0; + + static void Register(const std::string& name, + std::function()> factory); +}; + +enum class AdmissionDecision { + kAdmit, // 允许进入 + kRejectFrequency, // 频率不足 + kRejectWatermark, // 水位过高 + kRejectPrefix, // 前缀匹配不足 + kDefer, // 延迟决策 (记录候选) +}; + +// 已有实现: FrequencyAdmission (CountMinSketch) +// 新增实现: PrefixMatchAdmission, CostAwareAdmission +``` + +### 5.4.6.4 Ops 注册机制 + +``` +classDiagram + class EvictionOps { + <> + +SelectEvictionCandidates(snapshot, tier, bytes) vector~string~ + +Register(name, factory) void + } + class PrefetchOps { + <> + +GeneratePrefetchPlan(snapshot, trace) vector~PrefetchCandidate~ + +ExecutePrefetch(candidates, strategy) ErrorCode + +Register(name, factory) void + } + class AdmissionOps { + <> + +CheckAdmission(key, tier, snapshot) AdmissionDecision + +Register(name, factory) void + } + class PolicyEngine { + -ops_registry_ : map + +SelectOps(type, name) Ops + +ExecutePolicy(snapshot) PolicyResult + } + + PolicyEngine --> EvictionOps : 查找/执行 + PolicyEngine --> PrefetchOps : 查找/执行 + PolicyEngine --> AdmissionOps : 查找/执行 +``` + +## 5.4.7 写路径 IO Pattern + +### 5.4.7.1 写路径采集指标 + +| 指标名 | 类型 | 描述 | 采集来源 | +| ------------------- | ------ | ------------------------------ | ------------------- | +| `write_batch_size` | uint32 | 批量写入 key 数 | SDK 层 (`BatchPut`) | +| `write_object_size` | uint64 | 单次写入数据大小 | SDK 层 | +| `write_burst` | bool | 是否突发写入(短时间大量 Put) | 分析层计算 | +| `write_frequency` | uint32 | key 写入频率 | SDK 层 | +| `overwrite_ratio` | float | 覆盖写比例 (同 key 重复 Put) | 分析层计算 | + +### 5.4.7.2 写路径 Pattern 对策略的影响 + +| Pattern | 影响策略 | 处理方式 | +| ---------- | ---------------------- | ------------------------------------------------------------------ | +| 突发写入 | 准入:避免写穿 SSD | 突发写入期间提高 `admission_threshold`,冷数据暂留 DRAM 不 offload | +| 高覆盖写 | 准入:跳过 SSD offload | 覆盖写比例高的 key 不 offload 到 SSD,避免无效写入 | +| 大批量写入 | GC:提前触发 | 预估写入量,提前通知 SSD 后端准备 GC 空间 | +| 低频写入 | 淘汰:降低保留优先级 | 低频写入的 key 在淘汰评分中 `frequency` 低,优先淘汰 | + +## 5.4.8 健壮性与可观测性 + +### 5.4.8.1 失败降级 + +IO Pattern 模块自身故障时,必须不影响数据路径,降级到现有基础机制: + +``` +flowchart TD + START["策略执行请求"] + C1{"IO Pattern 模块可用?"} + C2{"分析层响应
超时?"} + NORMAL["正常路径:
ScoreBasedEviction / PrefixMatchAdmission / TraceBasedPrefetch"] + DEGRADE["降级路径:
LRU / FIFO / FrequencyAdmission
(现有基础机制)"] + + START --> C1 + C1 -->|是| C2 + C1 -->|否| DEGRADE + C2 -->|否| NORMAL + C2 -->|是| DEGRADE +``` + +| 故障场景 | 降级行为 | 触发条件 | +| -------------- | --------------------------------- | ------------------- | +| SubMaster 崩溃 | 回退到 Client 本地 LRU/FIFO | RPC 连续失败 > 3 次 | +| 分析层超时 | 使用上一次成功快照 | 响应延迟 > 500ms | +| 分析层 OOM | 丢弃 per-key 指标,仅保留全局指标 | 内存占用 > 阈值 | +| RPC 网络抖动 | 延长上报间隔,本地缓存策略 | 丢包率 > 5% | + +### 5.4.8.2 反馈闭环 + +策略执行后需评估效果并自适应调优参数,形成闭环: + +``` +flowchart LR + EXEC["策略执行
(eviction/prefetch/admission)"] + EVAL["效果评估
(命中率/eviction抖动/TTFT)"] + TUNE["参数调优
(权重/阈值自适应)"] + EXEC --> EVAL --> TUNE --> EXEC +``` + +**效果评估指标:** + +| 指标 | 描述 | 评估窗口 | +| ------------------- | ------------------------------ | ------------- | +| `hit_rate_delta` | 策略执行后命中率变化 | 60s 滑动窗口 | +| `eviction_churn` | 淘汰抖动(刚淘汰又被访问) | 120s 滑动窗口 | +| `ttft_delta` | TTFT 时延变化 | 30s 滑动窗口 | +| `prefetch_accuracy` | 预取命中率(预取后是否被访问) | 60s 滑动窗口 | + +**参数自适应:** 当 `hit_rate_delta < 0` 持续超过 3 个评估窗口时,自动回退权重调整(如降低 `α` 权重),或切换到更保守的策略(如 ScoreBased → LRU)。 + +### 5.4.8.3 IO Pattern 自观测 + +IO Pattern 模块自身的运行指标,用于运维和调优: + +| 指标 | 描述 | +| -------------------------------- | ----------------------------------------- | +| `io_pattern_collect_latency_us` | 单次采集延迟 | +| `io_pattern_analyze_latency_us` | 单次分析延迟 | +| `io_pattern_policy_decision_qps` | 策略决策 QPS | +| `io_pattern_strategy_hit_rate` | 策略命中率(策略命中 vs 总决策) | +| `io_pattern_false_positive_rate` | 误判率(预取未被访问 / 淘汰后被重新加载) | +| `io_pattern_degrade_count` | 降级次数 | +| `io_pattern_report_drop_count` | 上报丢弃数(采样降级) | + + +# IO Pattern implementation design + +This page records the implementation state of the IO Pattern design and is +updated together with the code. It is intentionally separate from the original +proposal so that unresolved decisions are visible. + +## Current architecture + +```text +Store/Get/Put -> Collector -> bounded Analyzer -> PolicyEngine -> Ops + | | |-> Eviction handler + | | |-> Prefetch handler + | | `-> Admission handler + | `-> per-session K-means fallback + `-> Reporter -> authenticated CFM channel/pool +``` + +`MasterService` owns the runtime because it owns the authoritative replica +metadata. Its handlers use the existing safe quota-eviction and +promotion-on-hit queues; HBM stays inference-runtime-owned and is never moved +by the Store master. + +## Implemented + +- `IoPatternCollectorImpl` aggregates inference, access and storage metrics by + tenant/object and returns deterministic snapshots. +- `ThresholdAnalyzer` classifies Code Agent, recommendation, conversation and + mixed workloads and calculates continuous confidence scores. +- `ScoreBasedEvictionOps`, `PrefixMatchAdmissionOps` and + `TraceBasedPrefetchOps` provide the first production policy implementations. +- `WorkloadPolicyEngine` selects workload templates, applies real weighted + transition over three detection windows, and selects independent templates + for K-means-labelled sessions. +- `OpsRegistry` and `RegistryPolicyEngine` resolve named policy implementations. +- `PolicyEngine::ExecutePolicy` returns one `PolicyResult` containing eviction, + prefetch and admission outcomes. +- `IoPatternReporter` provides bounded, non-blocking batches with explicit + report/drop counters and a transport-agnostic sink. +- `MetricBatchTransport` defines the transport seam, and the reporter exposes + load-sensitive 100/500/1000 ms flush recommendations. +- `IoPatternRuntime` wires collection, bounded analysis, policy execution, + feedback tuning and storage handlers; `MasterService` feeds it from actual + Get/Put/watermark paths. +- `CfmClientImpl` dispatches received policy commands through + `IoPatternRuntime::ExecuteCommand`, so CFM-issued plans take the same safe + Store execution route as locally planned ones. +- `CfmIngress` is the CFM-to-Store endpoint: it decodes authenticated snapshot + and metric-batch payloads into the runtime, and executes remote prefetch + plans through the same handlers. +- `ResilientCfmChannel` adds bounded retries and consecutive-failure + degradation state around a concrete transport. +- `PolicyFeedbackWindow` aggregates bounded execution-effect windows, and + `AdaptivePolicyTuner` adjusts eviction weights after repeated negative + hit-rate deltas. +- `IoPatternObservability` provides thread-safe counters for collection and + analysis latency, policy hit rate, false positives, degradation and report + drops. +- Its windowed snapshot also exposes strategy hit rate, false-positive rate and + policy decision QPS. +- `SlidingWindowAnalyzer` keeps timestamp-bounded snapshots and computes + median/p90 workload features before threshold classification. +- `IoPatternCollectorImpl` enforces an optional per-tenant key quota and + exposes dropped-observation counts for overload protection. +- The vLLM connector accumulates match, allocation and completion metrics per + request and reports a complete layout-aware record through its optional + `io_pattern_bridge`. `SglangHiCacheIoPatternBridge` provides the matching + bounded, layout-aware adapter for HiCache request-finished/prefix hooks. +- `TierOperationExecutor` bridges `PolicyResult` to storage-owned eviction, + prefetch and admission handlers and marks missing handlers as degraded. +- `ResilientAnalyzer` caches the last successful result and falls back to it + (or conservative mixed mode) when analysis throws, with failure tracking. +- `CfmBinaryCodec` defines the versioned `CFM2` protocol and fully round-trips + snapshots, metric batches and every policy command. `InProcessCfmRpcTransport` + provides authenticated embedded operation, while `CfmChannelPool` reuses and + fails over a bounded set of injected network channels. +- `CfmRpcChannel::SendMetricBatch` and `MakeCfmMetricBatchSink` connect the + bounded Reporter to the RPC path; producers only enqueue and Flush performs + the transport call outside the data-path critical section. +- `IoPatternReporter::Start/Stop` provides a background flush worker with + adaptive intervals; `Stop` performs a final synchronous drain. +- `IoPatternCollectorImpl` derives write-path fields for PUT records: + frequency, batch size, object size, overwrite ratio and burst flag. +- `DegradingPolicyEngine` switches to a caller-provided fallback engine after + repeated failures and supports explicit recovery. +- `AdaptivePolicyTuner` also reacts to eviction churn, TTFT regression and + prefetch accuracy, exposes conservative mode and supports persistence + callbacks for tuned weights. The runtime accepts feedback samples and + applies the resulting weights to both global and per-session engines. +- Analyzer execution has a single in-flight worker, timeout fallback to the + last safe result, and an explicit key-count budget; collector key quotas and + reporter bounds provide the associated overload/OOM protection. + +## Interface decision: complete plans versus document shorthand + +The proposal's shorthand methods returned only keys, candidates or a decision. +The implementation also needs tenant identity, byte sizes, scores, confidence, +timeout and strategy metadata. Therefore the complete plan interfaces are the +only Ops execution seam: + +- `EvictionOps::Evaluate` returns `EvictionPlan`. +- `PrefetchOps::Evaluate` returns `PrefetchPlan`. +- `AdmissionOps::Evaluate` returns `AdmissionResult`. + +The former shorthand methods (`SelectEvictionCandidates`, +`GeneratePrefetchPlan`, `ExecutePrefetch`, and `CheckAdmission`) have been +removed from the C++ interfaces. Callers must use complete plans and +`TierOperationExecutor` for execution. + +### Revised Ops contract (2026-09) + +The following contract supersedes the shorthand signatures in section 5.4.6: + +```cpp +class EvictionOps { + public: + virtual EvictionPlan Evaluate(const PolicyContext&, CacheTier, + uint64_t target_bytes) const = 0; +}; + +class PrefetchOps { + public: + virtual PrefetchPlan Evaluate(const PolicyContext&, + const TraceHistory&) const = 0; +}; + +class AdmissionOps { + public: + virtual AdmissionResult Evaluate(const ObjectRef&, CacheTier, + const PolicyContext&) const = 0; +}; +``` + +`EvictionPlan` carries tenant-qualified objects, byte budgets and scores; +`PrefetchPlan` carries source/target tiers, strategy, timeout and confidence; +`AdmissionResult` carries tenant identity, target tier, decision and +confidence. These fields are required by execution, observability and +multi-tenant isolation and must not be collapsed into strings. + +`PolicyEngine::ExecutePolicy` is the single orchestration entry point and +returns `PolicyResult` with explicit `degraded` propagation. + +Registry ownership is external and thread-safe. Factories return independent +Ops instances; callers own the returned smart pointers. Concrete storage and +RPC resources are injected through execution handlers and CFM channels. + +## Known gaps + +There are no remaining implementation gaps in the Mooncake IO Pattern scope. +Production deployments select their network-specific `CfmRpcTransport` through +the documented transport seam; the authenticated embedded transport is the +reference implementation and the SGLang adapter is intentionally kept +framework-neutral because SGLang source is not vendored in this repository. diff --git a/mooncake-store/include/cache_view_manager.h b/mooncake-store/include/cache_view_manager.h deleted file mode 100644 index 0a96be6fec..0000000000 --- a/mooncake-store/include/cache_view_manager.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include "io_pattern/view_manager.h" diff --git a/mooncake-store/include/cfm_client_impl.h b/mooncake-store/include/cfm_client_impl.h new file mode 100644 index 0000000000..da6bbcfe9c --- /dev/null +++ b/mooncake-store/include/cfm_client_impl.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/cfm_client_impl.h" diff --git a/mooncake-store/include/collector_impl.h b/mooncake-store/include/collector_impl.h new file mode 100644 index 0000000000..f85dc274e6 --- /dev/null +++ b/mooncake-store/include/collector_impl.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/collector_impl.h" diff --git a/mooncake-store/include/degrading_policy_engine.h b/mooncake-store/include/degrading_policy_engine.h new file mode 100644 index 0000000000..083068256e --- /dev/null +++ b/mooncake-store/include/degrading_policy_engine.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/degrading_policy_engine.h" diff --git a/mooncake-store/include/feedback.h b/mooncake-store/include/feedback.h new file mode 100644 index 0000000000..fc5b0e0b5b --- /dev/null +++ b/mooncake-store/include/feedback.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/feedback.h" diff --git a/mooncake-store/include/io_pattern/cfm_channel.h b/mooncake-store/include/io_pattern/cfm_channel.h new file mode 100644 index 0000000000..5533e113f0 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_channel.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "types.h" +#include "../types.h" + +namespace mooncake::io_pattern { + +// Transport-neutral CFM RPC channel. Implementations own serialization, +// retries and connection lifecycle. +class CfmChannel { + public: + virtual ~CfmChannel() = default; + virtual bool SendSnapshot(const IoPatternSnapshot& snapshot) = 0; + virtual std::optional PollPolicy() = 0; + virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_client_impl.h b/mooncake-store/include/io_pattern/cfm_client_impl.h new file mode 100644 index 0000000000..64ef8e2078 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_client_impl.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include "cfm_channel.h" +#include "client.h" + +namespace mooncake::io_pattern { + +// Production CFM client orchestration. Network behavior is delegated to the +// injected channel so this class remains independent of RPC libraries. +class CfmClientImpl final : public CfmClient { + public: + using PolicyCommandHandler = std::function; + + explicit CfmClientImpl(std::shared_ptr channel, + PolicyCommandHandler policy_handler = {}) + : channel_(std::move(channel)), + policy_handler_(std::move(policy_handler)) {} + + ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) override; + ErrorCode ReceivePolicy(const PolicyCommand& command) override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + std::optional PollPolicy(); + ErrorCode PollAndDispatchPolicy(); + + private: + std::shared_ptr channel_; + PolicyCommandHandler policy_handler_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_ingress.h b/mooncake-store/include/io_pattern/cfm_ingress.h new file mode 100644 index 0000000000..eb027e6575 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_ingress.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "cfm_protocol.h" +#include "runtime.h" + +namespace mooncake::io_pattern { + +// Server-side counterpart of CfmRpcChannel. Bind Handle() as an +// InProcessCfmRpcTransport::SendHandler or adapt it to a network RPC server. +class CfmIngress final { + public: + explicit CfmIngress(std::shared_ptr runtime, + std::shared_ptr codec = + std::make_shared()) + : runtime_(std::move(runtime)), codec_(std::move(codec)) {} + + bool Handle(std::string_view method, std::string_view payload); + + private: + std::shared_ptr runtime_; + std::shared_ptr codec_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_protocol.h b/mooncake-store/include/io_pattern/cfm_protocol.h new file mode 100644 index 0000000000..05d079a760 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_protocol.h @@ -0,0 +1,22 @@ +#pragma once + +#include "rpc_transport.h" + +namespace mooncake::io_pattern { + +// Versioned binary wire codec for the CFM RPC methods. It deliberately owns +// every serialization detail so transports only deal in authenticated bytes. +class CfmBinaryCodec final : public CfmRpcCodec { + public: + std::string EncodeSnapshot(const IoPatternSnapshot& snapshot) const override; + std::string EncodePrefetch(const PrefetchPlan& plan) const override; + std::string EncodeMetricBatch(const MetricBatch& batch) const override; + std::optional DecodePolicy(const std::string& payload) const override; + + std::optional DecodeSnapshot( + const std::string& payload) const; + std::optional DecodeMetricBatch(const std::string& payload) const; + std::string EncodePolicy(const PolicyCommand& command) const; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector.h b/mooncake-store/include/io_pattern/collector.h index 986a477b8f..be43bb69ae 100644 --- a/mooncake-store/include/io_pattern/collector.h +++ b/mooncake-store/include/io_pattern/collector.h @@ -11,7 +11,8 @@ class IoPatternCollector { // Implementations must not block the caller on RPC or storage I/O. virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; - virtual void RecordAccess(const AccessRecord& record) = 0; + virtual void RecordAccess(const std::string& key, + const AccessRecord& record) = 0; virtual void RecordStorageMetric(const StorageMetric& metric) = 0; virtual IoPatternSnapshot GetSnapshot() const = 0; }; diff --git a/mooncake-store/include/io_pattern/collector_impl.h b/mooncake-store/include/io_pattern/collector_impl.h new file mode 100644 index 0000000000..4324360750 --- /dev/null +++ b/mooncake-store/include/io_pattern/collector_impl.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include + +#include "collector.h" +#include "reporter.h" + +namespace mooncake::io_pattern { + +// Production collector that aggregates observations by tenant and object. +// It owns only metrics state; reporting to CFM is intentionally external. +class IoPatternCollectorImpl final : public IoPatternCollector { + public: + struct Config { + size_t max_keys_per_tenant{0}; + size_t max_total_keys{0}; + }; + + explicit IoPatternCollectorImpl(Config config = {}, + std::shared_ptr reporter = + nullptr) + : config_(config), reporter_(std::move(reporter)) {} + void ReportInferenceMetrics(const InferenceMetrics& metrics) override; + void RecordAccess(const std::string& key, + const AccessRecord& record) override; + void RecordStorageMetric(const StorageMetric& metric) override; + // Ingests a CFM snapshot without replaying it through the asynchronous + // reporter. The sender is already the reporting side of that pipeline. + void MergeSnapshot(const IoPatternSnapshot& snapshot); + IoPatternSnapshot GetSnapshot() const override; + uint64_t dropped() const; + bool degraded() const; + bool FlushReports(); + + private: + struct StorageMetricKey { + std::string source_id; + CacheTier tier{CacheTier::kL2Segment}; + bool operator==(const StorageMetricKey&) const = default; + }; + struct StorageMetricKeyHash { + size_t operator()(const StorageMetricKey& key) const noexcept { + return std::hash{}(key.source_id) ^ + (static_cast(key.tier) << 1); + } + }; + + mutable std::mutex mutex_; + Config config_; + std::shared_ptr reporter_; + uint64_t dropped_{0}; + bool degraded_{false}; + std::unordered_map key_metrics_; + std::unordered_map write_counts_; + std::unordered_map overwrite_counts_; + std::unordered_map tenant_key_counts_; + std::unordered_map + storage_metrics_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/degrading_policy_engine.h b/mooncake-store/include/io_pattern/degrading_policy_engine.h new file mode 100644 index 0000000000..f02e1d5b4c --- /dev/null +++ b/mooncake-store/include/io_pattern/degrading_policy_engine.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include "policy_engine.h" + +namespace mooncake::io_pattern { + +// Switches from a primary policy engine to a caller-provided fallback after +// repeated failures; recovery is explicit to avoid policy oscillation. +class DegradingPolicyEngine final : public PolicyEngine { + public: + DegradingPolicyEngine(std::shared_ptr primary, + std::shared_ptr fallback, + size_t failure_threshold = 3) + : primary_(std::move(primary)), + fallback_(std::move(fallback)), + failure_threshold_(failure_threshold) {} + + void RecordFailure(); + void RecordSuccess(); + void ForceDegraded(bool degraded); + bool degraded() const; + size_t consecutive_failures() const; + + EvictionPlan PlanEviction(const PolicyContext&, CacheTier, uint64_t) const override; + PrefetchPlan PlanPrefetch(const PolicyContext&, const TraceHistory&) const override; + AdmissionResult DecideAdmission(const ObjectRef&, CacheTier, + const PolicyContext&) const override; + + private: + std::shared_ptr Active() const; + mutable std::mutex mutex_; + std::shared_ptr primary_; + std::shared_ptr fallback_; + size_t failure_threshold_; + size_t consecutive_failures_{0}; + bool degraded_{false}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/feedback.h b/mooncake-store/include/io_pattern/feedback.h new file mode 100644 index 0000000000..9801e3a7ed --- /dev/null +++ b/mooncake-store/include/io_pattern/feedback.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "policy_strategies.h" + +namespace mooncake::io_pattern { + +struct PolicyFeedbackSample { + float hit_rate_delta{0.0F}; + float eviction_churn{0.0F}; + float ttft_delta{0.0F}; + float prefetch_accuracy{0.0F}; +}; + +struct PolicyFeedbackStats { + float hit_rate_delta{0.0F}; + float eviction_churn{0.0F}; + float ttft_delta{0.0F}; + float prefetch_accuracy{0.0F}; + size_t samples{0}; +}; + +class PolicyFeedbackWindow final { + public: + explicit PolicyFeedbackWindow(size_t capacity = 60) : capacity_(capacity) {} + void Record(PolicyFeedbackSample sample); + PolicyFeedbackStats Snapshot() const; + + private: + const size_t capacity_; + mutable std::mutex mutex_; + std::deque samples_; +}; + +// Conservative tuner: after three consecutive negative hit-rate windows, +// reduce frequency weight and increase idle weight to curb cache churn. +class AdaptivePolicyTuner final { + public: + explicit AdaptivePolicyTuner(size_t negative_windows = 3) + : negative_windows_(negative_windows) {} + bool Tune(const PolicyFeedbackStats& stats, + ScoreBasedEvictionConfig& config); + bool conservative() const { return conservative_; } + void SetPersistenceCallback(std::function + callback) { + persistence_ = std::move(callback); + } + + private: + const size_t negative_windows_; + size_t negative_streak_{0}; + bool conservative_{false}; + std::function persistence_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 4ffbcaa5f1..0bc97def1d 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -2,9 +2,27 @@ #include "io_pattern/analyzer.h" #include "io_pattern/client.h" +#include "io_pattern/cfm_channel.h" +#include "io_pattern/cfm_ingress.h" +#include "io_pattern/cfm_protocol.h" +#include "io_pattern/cfm_client_impl.h" +#include "io_pattern/feedback.h" +#include "io_pattern/degrading_policy_engine.h" +#include "io_pattern/legacy_eviction_ops.h" +#include "io_pattern/kmeans_analyzer.h" #include "io_pattern/collector.h" +#include "io_pattern/collector_impl.h" #include "io_pattern/ops.h" +#include "io_pattern/observability.h" #include "io_pattern/policy_engine.h" +#include "io_pattern/reporter.h" +#include "io_pattern/resilient_analyzer.h" +#include "io_pattern/rpc_transport.h" +#include "io_pattern/runtime.h" +#include "io_pattern/sliding_window_analyzer.h" +#include "io_pattern/resilient_cfm_channel.h" +#include "io_pattern/policy_strategies.h" #include "io_pattern/registry.h" #include "io_pattern/types.h" -#include "io_pattern/view_manager.h" +#include "io_pattern/threshold_analyzer.h" +#include "io_pattern/tier_executor.h" diff --git a/mooncake-store/include/io_pattern/kmeans_analyzer.h b/mooncake-store/include/io_pattern/kmeans_analyzer.h new file mode 100644 index 0000000000..e8962e5346 --- /dev/null +++ b/mooncake-store/include/io_pattern/kmeans_analyzer.h @@ -0,0 +1,28 @@ +#pragma once + +#include "threshold_analyzer.h" + +namespace mooncake::io_pattern { + +// Slow-path workload detector for mixed traffic. It clusters session feature +// vectors, then maps each centroid to the documented workload templates. +class KMeansWorkloadAnalyzer final : public IoPatternAnalyzer { + public: + struct Config { + uint32_t iterations{8}; + ThresholdAnalyzerConfig thresholds{}; + }; + + explicit KMeansWorkloadAnalyzer(Config config = {}) : config_(config) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + + private: + Config config_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/legacy_eviction_ops.h b/mooncake-store/include/io_pattern/legacy_eviction_ops.h new file mode 100644 index 0000000000..5145b8bcc1 --- /dev/null +++ b/mooncake-store/include/io_pattern/legacy_eviction_ops.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "../eviction_strategy.h" +#include "ops.h" + +namespace mooncake::io_pattern { + +class LegacyEvictionOps final : public EvictionOps { + public: + explicit LegacyEvictionOps(std::shared_ptr strategy) + : strategy_(std::move(strategy)) {} + + EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override; + + private: + std::shared_ptr strategy_; + mutable std::mutex mutex_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/observability.h b/mooncake-store/include/io_pattern/observability.h new file mode 100644 index 0000000000..2608640686 --- /dev/null +++ b/mooncake-store/include/io_pattern/observability.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +namespace mooncake::io_pattern { + +struct IoPatternObservabilitySnapshot { + uint64_t collect_latency_us{0}; + uint64_t analyze_latency_us{0}; + uint64_t policy_decisions{0}; + uint64_t strategy_hits{0}; + uint64_t strategy_trials{0}; + uint64_t false_positives{0}; + uint64_t degrade_count{0}; + uint64_t report_drop_count{0}; + float strategy_hit_rate{0.0F}; + float false_positive_rate{0.0F}; + float policy_decision_qps{0.0F}; +}; + +// Thread-safe counters for IO Pattern operational metrics. +class IoPatternObservability final { + public: + void RecordCollectLatency(uint64_t latency_us); + void RecordAnalyzeLatency(uint64_t latency_us); + void RecordPolicyDecision(bool strategy_hit); + void RecordFalsePositive(); + void RecordDegrade(); + void RecordReportDrop(uint64_t count = 1); + IoPatternObservabilitySnapshot Snapshot() const; + IoPatternObservabilitySnapshot Snapshot(double window_seconds) const; + + private: + mutable std::mutex mutex_; + IoPatternObservabilitySnapshot values_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/ops.h b/mooncake-store/include/io_pattern/ops.h index 37e6c64941..fbf57ac869 100644 --- a/mooncake-store/include/io_pattern/ops.h +++ b/mooncake-store/include/io_pattern/ops.h @@ -12,8 +12,10 @@ class EvictionOps { public: virtual ~EvictionOps() = default; + // Returns a complete plan carrying tenant, score and byte metadata. virtual EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, uint64_t target_bytes) const = 0; + }; // Produces a prefetch plan; execution belongs to PrefetchExecutor. @@ -21,6 +23,7 @@ class PrefetchOps { public: virtual ~PrefetchOps() = default; + // Returns a complete plan carrying strategy, timeout and confidence. virtual PrefetchPlan Evaluate(const PolicyContext& context, const TraceHistory& trace) const = 0; }; @@ -30,6 +33,7 @@ class AdmissionOps { public: virtual ~AdmissionOps() = default; + // Returns a complete decision carrying tenant identity and confidence. virtual AdmissionResult Evaluate(const ObjectRef& object, CacheTier target_tier, const PolicyContext& context) const = 0; diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index 4d419b1a72..afa8865903 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -1,10 +1,17 @@ #pragma once +#include #include #include +#include +#include +#include #include +#include #include "io_pattern/ops.h" +#include "io_pattern/policy_strategies.h" +#include "io_pattern/registry.h" #include "io_pattern/types.h" namespace mooncake::io_pattern { @@ -22,6 +29,22 @@ class PolicyEngine { virtual AdmissionResult DecideAdmission( const ObjectRef& object, CacheTier target_tier, const PolicyContext& context) const = 0; + + // Executes the three policy dimensions through one uniform result seam. + virtual PolicyResult ExecutePolicy(const PolicyContext& context, + CacheTier eviction_tier, + uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions = {}) const { + PolicyResult result; + result.eviction = PlanEviction(context, eviction_tier, eviction_bytes); + result.prefetch = PlanPrefetch(context, trace); + for (const auto& object : admissions) { + result.admissions.push_back( + DecideAdmission(object, eviction_tier, context)); + } + return result; + } }; // A small composition adapter that wires selected Ops instances together. @@ -68,4 +91,262 @@ class ComposedPolicyEngine final : public PolicyEngine { std::shared_ptr admission_; }; +// Resolves Ops implementations by registry name and composes them for one +// policy execution. Factories are consulted per call to avoid shared state. +class RegistryPolicyEngine final : public PolicyEngine { + public: + RegistryPolicyEngine(std::shared_ptr registries, + std::string eviction_name, + std::string prefetch_name, + std::string admission_name) + : registries_(std::move(registries)), + eviction_name_(std::move(eviction_name)), + prefetch_name_(std::move(prefetch_name)), + admission_name_(std::move(admission_name)) {} + + EvictionPlan PlanEviction(const PolicyContext& context, CacheTier tier, + uint64_t bytes) const override { + auto engine = Compose(); + return engine->PlanEviction(context, tier, bytes); + } + PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const override { + return Compose()->PlanPrefetch(context, trace); + } + AdmissionResult DecideAdmission(const ObjectRef& object, CacheTier tier, + const PolicyContext& context) const override { + return Compose()->DecideAdmission(object, tier, context); + } + + PolicyResult ExecutePolicy(const PolicyContext& context, + CacheTier tier, uint64_t bytes, + const TraceHistory& trace, + const std::vector& admissions = {}) const override { + auto result = PolicyEngine::ExecutePolicy(context, tier, bytes, trace, + admissions); + std::shared_lock lock(mutex_); + result.degraded = !registries_ || + !registries_->eviction.Create(eviction_name_) || + !registries_->prefetch.Create(prefetch_name_) || + !registries_->admission.Create(admission_name_); + return result; + } + + private: + std::shared_ptr Compose() const { + if (!registries_) return std::make_shared(nullptr, nullptr, nullptr); + return std::make_shared( + registries_->eviction.Create(eviction_name_), + registries_->prefetch.Create(prefetch_name_), + registries_->admission.Create(admission_name_)); + } + + std::shared_ptr registries_; + mutable std::shared_mutex mutex_; + std::string eviction_name_; + std::string prefetch_name_; + std::string admission_name_; +}; + +// Selects the documented policy template for the current workload. +class WorkloadPolicyEngine final : public PolicyEngine { + public: + explicit WorkloadPolicyEngine(WorkloadType type = WorkloadType::kMixed, + uint32_t transition_windows = 3) + : transition_windows_(transition_windows), workload_type_(type), + previous_type_(type) { + Configure(type); + } + + void SetWorkloadType(WorkloadType type) { + std::unique_lock lock(mutex_); + if (type == workload_type_) return; + previous_type_ = workload_type_; + previous_eviction_ = active_eviction_; + workload_type_ = type; + transition_progress_ = transition_windows_ == 0 ? 1.0F : 0.0F; + Configure(type); + } + + WorkloadType ActiveWorkload() const { + std::shared_lock lock(mutex_); + return workload_type_; + } + + float TransitionProgress() const { + std::shared_lock lock(mutex_); + return transition_progress_; + } + + // Advances the template transition by one completed detection window. + void AdvanceTransitionWindow() { + std::unique_lock lock(mutex_); + if (transition_progress_ < 1.0F && transition_windows_ != 0) { + transition_progress_ = std::min( + 1.0F, transition_progress_ + 1.0F / + static_cast(transition_windows_)); + } + } + + void SetSessionWorkloads(const std::vector& sessions) { + std::unique_lock lock(mutex_); + session_engines_.clear(); + session_types_.clear(); + for (const auto& session : sessions) { + if (!session.session_id.empty()) { + session_engines_[session.session_id] = MakeEngine(session.workload_type); + session_types_[session.session_id] = session.workload_type; + } + } + } + + ScoreBasedEvictionConfig CurrentEvictionConfig() const { + std::shared_lock lock(mutex_); + return active_eviction_; + } + + void ApplyEvictionTuning(const ScoreBasedEvictionConfig& config) { + std::unique_lock lock(mutex_); + tuned_eviction_ = config; + Configure(workload_type_); + for (auto& [session, engine] : session_engines_) { + engine = MakeEngine(session_types_.at(session)); + } + } + + EvictionPlan PlanEviction(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override { + std::shared_lock lock(mutex_); + if (context.session_id.empty() && transition_progress_ < 1.0F) { + auto blended = MakeEngine( + workload_type_, Blend(previous_eviction_, active_eviction_, + transition_progress_)); + return blended->PlanEviction(context, tier, target_bytes); + } + return SelectEngine(context)->PlanEviction(context, tier, target_bytes); + } + PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const override { + std::shared_lock lock(mutex_); + return SelectEngine(context)->PlanPrefetch(context, trace); + } + AdmissionResult DecideAdmission(const ObjectRef& object, + CacheTier tier, + const PolicyContext& context) const override { + std::shared_lock lock(mutex_); + return SelectEngine(context)->DecideAdmission(object, tier, context); + } + + private: + void Configure(WorkloadType type) { + engine_ = MakeEngine(type); + active_eviction_ = EvictionConfigFor(type); + if (tuned_eviction_) active_eviction_ = *tuned_eviction_; + } + + ScoreBasedEvictionConfig EvictionConfigFor(WorkloadType type) const { + ScoreBasedEvictionConfig eviction; + switch (type) { + case WorkloadType::kCodeAgent: + eviction.idle_weight = 0.8F; + eviction.frequency_weight = 0.2F; + eviction.prefix_weight = 0.3F; + eviction.tier_down_mode = TierDownMode::kSkipHost; + break; + case WorkloadType::kGenerativeRecommendation: + eviction.idle_weight = 0.3F; + eviction.frequency_weight = 0.8F; + eviction.prefix_weight = 0.2F; + eviction.tier_down_mode = TierDownMode::kStepwise; + break; + case WorkloadType::kMultiTurnConversation: + eviction.idle_weight = 0.5F; + eviction.frequency_weight = 0.4F; + eviction.prefix_weight = 0.6F; + eviction.tier_down_mode = TierDownMode::kPrefixAffinity; + break; + case WorkloadType::kUnknown: + case WorkloadType::kMixed: + break; + } + return eviction; + } + + static ScoreBasedEvictionConfig Blend(const ScoreBasedEvictionConfig& from, + const ScoreBasedEvictionConfig& to, + float progress) { + const auto blend = [progress](float old_value, float new_value) { + return old_value * (1.0F - progress) + new_value * progress; + }; + ScoreBasedEvictionConfig result = to; + result.idle_weight = blend(from.idle_weight, to.idle_weight); + result.frequency_weight = blend(from.frequency_weight, to.frequency_weight); + result.prefix_weight = blend(from.prefix_weight, to.prefix_weight); + result.recompute_weight = blend(from.recompute_weight, to.recompute_weight); + result.lower_replica_weight = + blend(from.lower_replica_weight, to.lower_replica_weight); + result.other_replica_weight = + blend(from.other_replica_weight, to.other_replica_weight); + // Tier routing is categorical, so switch at the midpoint while the + // numerical eviction weights transition continuously. + result.tier_down_mode = progress < 0.5F ? from.tier_down_mode + : to.tier_down_mode; + return result; + } + + std::shared_ptr MakeEngine( + WorkloadType type, + std::optional eviction_override = std::nullopt) const { + ScoreBasedEvictionConfig eviction = + eviction_override.value_or(EvictionConfigFor(type)); + PrefixMatchAdmissionConfig admission; + TraceBasedPrefetchConfig prefetch; + switch (type) { + case WorkloadType::kCodeAgent: + prefetch.match_length_threshold = 512; + break; + case WorkloadType::kGenerativeRecommendation: + admission.frequency_threshold = 20; + prefetch.match_length_threshold = 64; + prefetch.strategy = PrefetchStrategy::kWaitComplete; + break; + case WorkloadType::kMultiTurnConversation: + prefetch.match_length_threshold = 256; + prefetch.strategy = PrefetchStrategy::kTimeout; + prefetch.timeout_us = 5000; + break; + case WorkloadType::kUnknown: + case WorkloadType::kMixed: + break; + } + if (tuned_eviction_ && !eviction_override) eviction = *tuned_eviction_; + return std::make_shared( + std::make_shared(eviction), + std::make_shared(prefetch), + std::make_shared(admission)); + } + + std::shared_ptr SelectEngine( + const PolicyContext& context) const { + if (!context.session_id.empty()) { + const auto it = session_engines_.find(context.session_id); + if (it != session_engines_.end()) return it->second; + } + return engine_; + } + + mutable std::shared_mutex mutex_; + std::shared_ptr engine_; + std::unordered_map> + session_engines_; + std::unordered_map session_types_; + WorkloadType workload_type_{WorkloadType::kUnknown}; + WorkloadType previous_type_{WorkloadType::kUnknown}; + uint32_t transition_windows_{3}; + float transition_progress_{1.0F}; + std::optional tuned_eviction_; + ScoreBasedEvictionConfig active_eviction_; + ScoreBasedEvictionConfig previous_eviction_; +}; + } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/policy_strategies.h b/mooncake-store/include/io_pattern/policy_strategies.h new file mode 100644 index 0000000000..6ef274405d --- /dev/null +++ b/mooncake-store/include/io_pattern/policy_strategies.h @@ -0,0 +1,74 @@ +#pragma once + +#include "io_pattern/ops.h" + +namespace mooncake::io_pattern { + +enum class TierDownMode : uint8_t { + kStepwise, + kSkipHost, + kPrefixAffinity, +}; + +struct ScoreBasedEvictionConfig { + float idle_weight{1.0F}; + float frequency_weight{1.0F}; + float prefix_weight{1.0F}; + float recompute_weight{1.0F}; + float lower_replica_weight{1.0F}; + float other_replica_weight{1.0F}; + TierDownMode tier_down_mode{TierDownMode::kStepwise}; + uint64_t max_candidates{0}; +}; + +class ScoreBasedEvictionOps final : public EvictionOps { + public: + explicit ScoreBasedEvictionOps(ScoreBasedEvictionConfig config = {}) + : config_(config) {} + + EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override; + + private: + ScoreBasedEvictionConfig config_; +}; + +struct PrefixMatchAdmissionConfig { + uint32_t hbm_match_length{64}; + uint64_t frequency_threshold{1}; + float max_memory_used_ratio{0.90F}; +}; + +class PrefixMatchAdmissionOps final : public AdmissionOps { + public: + explicit PrefixMatchAdmissionOps(PrefixMatchAdmissionConfig config = {}) + : config_(config) {} + + AdmissionResult Evaluate(const ObjectRef& object, CacheTier target_tier, + const PolicyContext& context) const override; + + private: + PrefixMatchAdmissionConfig config_; +}; + +struct TraceBasedPrefetchConfig { + uint32_t match_length_threshold{256}; + float minimum_confidence{0.6F}; + uint64_t max_candidates{0}; + PrefetchStrategy strategy{PrefetchStrategy::kBestEffort}; + uint64_t timeout_us{0}; +}; + +class TraceBasedPrefetchOps final : public PrefetchOps { + public: + explicit TraceBasedPrefetchOps(TraceBasedPrefetchConfig config = {}) + : config_(config) {} + + PrefetchPlan Evaluate(const PolicyContext& context, + const TraceHistory& trace) const override; + + private: + TraceBasedPrefetchConfig config_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/reporter.h b/mooncake-store/include/io_pattern/reporter.h new file mode 100644 index 0000000000..fc0cbdf364 --- /dev/null +++ b/mooncake-store/include/io_pattern/reporter.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "types.h" + +namespace mooncake::io_pattern { + +struct MetricBatch { + std::vector inference; + std::vector accesses; + std::vector storage; +}; + +using MetricBatchSink = std::function; + +class MetricBatchTransport { + public: + virtual ~MetricBatchTransport() = default; + virtual bool Send(const MetricBatch& batch) = 0; +}; + +// Bounded, non-blocking metric batching. Transport and RPC ownership remain +// with the supplied sink. +class IoPatternReporter final { + public: + explicit IoPatternReporter(size_t capacity, MetricBatchSink sink, + size_t per_tenant_capacity = 0); + ~IoPatternReporter(); + + void Start(); + void Stop(); + + bool Enqueue(InferenceMetrics metrics); + bool EnqueueAccess(AccessRecord record); + bool EnqueueStorage(StorageMetric metric); + bool Flush(); + + size_t pending() const; + uint64_t dropped() const; + uint64_t reported() const; + std::chrono::milliseconds RecommendedFlushInterval() const; + + private: + bool EnqueueImpl(std::function append, + const TenantId& tenant); + + const size_t capacity_; + const MetricBatchSink sink_; + const size_t per_tenant_capacity_; + mutable std::mutex mutex_; + MetricBatch batch_; + uint64_t dropped_{0}; + uint64_t reported_{0}; + std::condition_variable condition_; + std::thread worker_; + bool running_{false}; + std::unordered_map tenant_pending_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/resilient_analyzer.h b/mooncake-store/include/io_pattern/resilient_analyzer.h new file mode 100644 index 0000000000..7f789d97b7 --- /dev/null +++ b/mooncake-store/include/io_pattern/resilient_analyzer.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +#include "analyzer.h" + +namespace mooncake::io_pattern { + +class ResilientAnalyzer final : public IoPatternAnalyzer { + public: + explicit ResilientAnalyzer(std::shared_ptr primary, + size_t failure_threshold = 3) + : primary_(std::move(primary)), failure_threshold_(failure_threshold) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + bool degraded() const; + size_t failures() const; + // Returns the most recent safe answer without invoking the primary + // analyzer. Used by a bounded caller when its analysis budget expires. + PatternResult FallbackResult() const; + + private: + void RecordFailure() const; + void RecordSuccess(const PatternResult& result) const; + PatternResult Fallback() const; + + std::shared_ptr primary_; + const size_t failure_threshold_; + mutable std::mutex mutex_; + mutable PatternResult last_result_; + mutable size_t failures_{0}; + mutable bool degraded_{false}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/resilient_cfm_channel.h b/mooncake-store/include/io_pattern/resilient_cfm_channel.h new file mode 100644 index 0000000000..d0b2118664 --- /dev/null +++ b/mooncake-store/include/io_pattern/resilient_cfm_channel.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include "cfm_channel.h" + +namespace mooncake::io_pattern { + +struct CfmRetryConfig { + uint32_t max_retries{3}; + uint32_t degrade_after_failures{3}; +}; + +// Adds bounded retry and health tracking to any concrete CFM transport. +class ResilientCfmChannel final : public CfmChannel { + public: + ResilientCfmChannel(std::shared_ptr delegate, + CfmRetryConfig config = {}) + : delegate_(std::move(delegate)), config_(config) {} + + bool SendSnapshot(const IoPatternSnapshot& snapshot) override; + std::optional PollPolicy() override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + bool degraded() const; + uint64_t consecutive_failures() const; + + private: + template + bool Retry(Operation&& operation); + void RecordSuccess(); + void RecordFailure(); + + std::shared_ptr delegate_; + CfmRetryConfig config_; + mutable std::mutex mutex_; + uint64_t consecutive_failures_{0}; + bool degraded_{false}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/rpc_transport.h b/mooncake-store/include/io_pattern/rpc_transport.h new file mode 100644 index 0000000000..234f99c1ec --- /dev/null +++ b/mooncake-store/include/io_pattern/rpc_transport.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cfm_channel.h" +#include "reporter.h" + +namespace mooncake::io_pattern { + +class CfmRpcCodec { + public: + virtual ~CfmRpcCodec() = default; + virtual std::string EncodeSnapshot(const IoPatternSnapshot&) const = 0; + virtual std::string EncodePrefetch(const PrefetchPlan&) const = 0; + virtual std::string EncodeMetricBatch(const MetricBatch&) const = 0; + virtual std::optional DecodePolicy( + const std::string&) const = 0; +}; + +class CfmRpcTransport { + public: + virtual ~CfmRpcTransport() = default; + // Implementations that communicate with a remote CFM should override this + // to bind the connection to the configured service credential. Keeping a + // default preserves compatibility with trusted in-process transports. + virtual bool Authenticate(std::string_view token) { return token.empty(); } + virtual bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) = 0; + virtual std::optional Receive( + std::string_view method, std::chrono::milliseconds timeout) = 0; +}; + +struct CfmRpcConfig { + std::chrono::milliseconds timeout{500}; + std::string auth_token; +}; + +// A concrete authenticated endpoint for embedded deployments and integration +// tests. It is intentionally transport-agnostic at the codec boundary: a +// socket/HTTP implementation can expose the same method names and wire bytes. +class InProcessCfmRpcTransport final : public CfmRpcTransport { + public: + using SendHandler = std::function; + + explicit InProcessCfmRpcTransport(std::string auth_token, + SendHandler send_handler = {}) + : auth_token_(std::move(auth_token)), send_handler_(std::move(send_handler)) {} + + bool Authenticate(std::string_view token) override; + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) override; + std::optional Receive( + std::string_view method, std::chrono::milliseconds timeout) override; + + void EnqueuePolicy(std::string payload); + void SetSendHandler(SendHandler handler); + + private: + mutable std::mutex mutex_; + const std::string auth_token_; + bool authenticated_{false}; + SendHandler send_handler_; + std::queue policies_; +}; + +class CfmRpcChannel final : public CfmChannel { + public: + CfmRpcChannel(std::shared_ptr transport, + std::shared_ptr codec, + CfmRpcConfig config = {}) + : transport_(std::move(transport)), + codec_(std::move(codec)), + config_(config) {} + + bool SendSnapshot(const IoPatternSnapshot& snapshot) override; + std::optional PollPolicy() override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + bool SendMetricBatch(const MetricBatch& batch); + + private: + bool EnsureAuthenticated(); + + std::shared_ptr transport_; + std::shared_ptr codec_; + CfmRpcConfig config_; + std::mutex authentication_mutex_; + bool authenticated_{false}; +}; + +// Reuses a bounded set of authenticated CFM channels. Requests are selected +// round-robin; an unavailable member is skipped so one failed connection does +// not stall policy reporting. +class CfmChannelPool final : public CfmChannel { + public: + explicit CfmChannelPool(std::vector> channels) + : channels_(std::move(channels)) {} + + bool SendSnapshot(const IoPatternSnapshot& snapshot) override; + std::optional PollPolicy() override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + private: + std::shared_ptr Next() const; + + std::vector> channels_; + mutable std::atomic next_{0}; +}; + +// Adapts the RPC channel to the reporter's asynchronous batch sink. +inline MetricBatchSink MakeCfmMetricBatchSink( + std::shared_ptr channel) { + return [channel = std::move(channel)](const MetricBatch& batch) { + return channel && channel->SendMetricBatch(batch); + }; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h new file mode 100644 index 0000000000..74acfec1bd --- /dev/null +++ b/mooncake-store/include/io_pattern/runtime.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "collector_impl.h" +#include "degrading_policy_engine.h" +#include "feedback.h" +#include "legacy_eviction_ops.h" +#include "observability.h" +#include "policy_engine.h" +#include "resilient_analyzer.h" +#include "sliding_window_analyzer.h" +#include "tier_executor.h" + +namespace mooncake::io_pattern { + +// Owns the Store-side IO Pattern pipeline. Producers only record observations; +// policy evaluation and storage operations run through this explicit runtime +// seam so collection never blocks the data path. +class IoPatternRuntime final { + public: + enum class LegacyFallback { kLru, kFifo }; + struct Handlers { + EvictionHandler eviction; + PrefetchHandler prefetch; + AdmissionHandler admission; + }; + + struct Config { + IoPatternCollectorImpl::Config collector; + uint64_t analysis_window_ns{60'000'000'000ULL}; + uint64_t analysis_timeout_us{500'000}; + size_t max_analysis_keys{100'000}; + size_t feedback_window{60}; + size_t report_capacity{4096}; + size_t report_per_tenant_capacity{0}; + size_t max_pending_prefetches{4096}; + MetricBatchSink report_sink; + LegacyFallback legacy_fallback{LegacyFallback::kLru}; + }; + + explicit IoPatternRuntime(Handlers handlers, Config config = {}); + ~IoPatternRuntime(); + + void ReportInferenceMetrics(const InferenceMetrics& metrics); + void RecordAccess(const std::string& key, const AccessRecord& record); + void RecordStorageMetric(const StorageMetric& metric); + void MergeSnapshot(const IoPatternSnapshot& snapshot); + + PolicyExecutionStatus Execute( + CacheTier eviction_tier, uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions = {}, + const std::string& session_id = {}); + // Applies a CFM-issued command through the same storage handlers as a + // locally planned policy. This is the CFM-to-Store execution endpoint. + ErrorCode ExecuteCommand(const PolicyCommand& command); + + void RecordFeedback(PolicyFeedbackSample sample); + IoPatternSnapshot Snapshot() const; + IoPatternObservabilitySnapshot ObservabilitySnapshot( + double window_seconds = 0.0) const; + bool degraded() const; + + private: + PatternResult AnalyzeWithinBudget(const IoPatternSnapshot& snapshot, + bool& degraded); + + Config config_; + std::shared_ptr reporter_; + std::shared_ptr collector_; + std::shared_ptr analyzer_; + std::shared_ptr workload_policy_; + std::shared_ptr policy_; + TierOperationExecutor executor_; + PolicyFeedbackWindow feedback_; + AdaptivePolicyTuner tuner_; + IoPatternObservability observability_; + mutable std::mutex feedback_state_mutex_; + std::unordered_set pending_prefetches_; + uint64_t feedback_accesses_{0}; + uint64_t feedback_hits_{0}; + float previous_hit_rate_{0.0F}; + // Shared with a timed-out detached analyzer so runtime teardown cannot + // leave a worker holding a pointer into a destroyed runtime instance. + std::shared_ptr> analysis_in_flight_{ + std::make_shared>(false)}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/sliding_window_analyzer.h b/mooncake-store/include/io_pattern/sliding_window_analyzer.h new file mode 100644 index 0000000000..583190d254 --- /dev/null +++ b/mooncake-store/include/io_pattern/sliding_window_analyzer.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include + +#include "threshold_analyzer.h" +#include "kmeans_analyzer.h" + +namespace mooncake::io_pattern { + +struct WorkloadFeatureStats { + uint32_t token_median{0}; + uint32_t token_p90{0}; + uint32_t fanout_p90{0}; + uint64_t block_median{0}; + uint64_t block_p90{0}; + uint32_t match_p90{0}; + uint32_t frequency_median{0}; + size_t samples{0}; +}; + +// Maintains a timestamp-bounded history of snapshots for workload detection. +class SlidingWindowAnalyzer final : public IoPatternAnalyzer { + public: + explicit SlidingWindowAnalyzer(uint64_t window_ns = 60'000'000'000ULL, + ThresholdAnalyzerConfig config = {}) + : window_ns_(window_ns), + analyzer_(config), + kmeans_(KMeansWorkloadAnalyzer::Config{.thresholds = config}) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + WorkloadFeatureStats FeatureStats() const; + + private: + IoPatternSnapshot Aggregate(const IoPatternSnapshot& current) const; + void Append(const IoPatternSnapshot& snapshot) const; + + const uint64_t window_ns_; + mutable std::mutex mutex_; + mutable std::deque history_; + ThresholdAnalyzer analyzer_; + KMeansWorkloadAnalyzer kmeans_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/threshold_analyzer.h b/mooncake-store/include/io_pattern/threshold_analyzer.h new file mode 100644 index 0000000000..2fd3baa0c4 --- /dev/null +++ b/mooncake-store/include/io_pattern/threshold_analyzer.h @@ -0,0 +1,38 @@ +#pragma once + +#include "io_pattern/analyzer.h" + +namespace mooncake::io_pattern { + +struct ThresholdAnalyzerConfig { + uint32_t code_agent_token_count{16 * 1024}; + uint32_t code_agent_prefix_fanout{16}; + uint32_t code_agent_match_length{256}; + uint64_t recommendation_block_size{128 * 1024}; + uint32_t recommendation_frequency{20}; + uint32_t conversation_prefix_fanout{16}; + uint32_t conversation_match_length{256}; +}; + +// Deterministic, low-latency analyzer for the documented threshold path. +// Mixed workloads are intentionally returned when no rule matches. +class ThresholdAnalyzer final : public IoPatternAnalyzer { + public: + explicit ThresholdAnalyzer(ThresholdAnalyzerConfig config = {}) + : config_(config) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + + private: + const KeyMetrics* FindKey(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const; + float KeyConfidence(const KeyMetrics& key) const; + + ThresholdAnalyzerConfig config_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/tier_executor.h b/mooncake-store/include/io_pattern/tier_executor.h new file mode 100644 index 0000000000..3ac0265d7f --- /dev/null +++ b/mooncake-store/include/io_pattern/tier_executor.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "types.h" + +namespace mooncake::io_pattern { + +using EvictionHandler = std::function; +using PrefetchHandler = std::function; +using AdmissionHandler = std::function; + +struct PolicyExecutionStatus { + ErrorCode eviction{ErrorCode::OK}; + ErrorCode prefetch{ErrorCode::OK}; + std::vector admissions; + bool degraded{false}; +}; + +// Bridges policy output to storage/tier mechanisms owned by other modules. +class TierOperationExecutor final { + public: + TierOperationExecutor(EvictionHandler eviction, + PrefetchHandler prefetch, + AdmissionHandler admission) + : eviction_(std::move(eviction)), + prefetch_(std::move(prefetch)), + admission_(std::move(admission)) {} + + PolicyExecutionStatus Execute(const PolicyResult& result) const; + + private: + EvictionHandler eviction_; + PrefetchHandler prefetch_; + AdmissionHandler admission_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index 1fb936e56d..503766e0f3 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -60,6 +60,15 @@ struct ObjectRef { bool operator==(const ObjectRef&) const = default; }; +struct ObjectRefHash { + size_t operator()(const ObjectRef& object) const noexcept { + const size_t tenant_hash = TenantIdHash{}(object.tenant_id); + const size_t key_hash = std::hash{}(object.key); + return tenant_hash ^ (key_hash + 0x9e3779b9 + (tenant_hash << 6) + + (tenant_hash >> 2)); + } +}; + struct InferenceMetrics { ObjectRef object; std::string session_id; @@ -82,6 +91,8 @@ struct AccessRecord { CacheTier tier{CacheTier::kL2Segment}; IoOperation operation{IoOperation::kGet}; bool is_hit{false}; + uint32_t write_batch_size{0}; + bool overwrite{false}; }; struct StorageMetric { @@ -101,6 +112,7 @@ struct StorageMetric { struct KeyMetrics { ObjectRef object; + std::string session_id; uint64_t last_access_time_ns{0}; uint64_t access_count_window{0}; uint64_t idle_time_us{0}; @@ -144,15 +156,23 @@ struct KeyPattern { bool migration_safe{false}; }; +struct SessionPattern { + std::string session_id; + WorkloadType workload_type{WorkloadType::kUnknown}; + float confidence{0.0F}; +}; + struct PatternResult { WorkloadType workload_type{WorkloadType::kUnknown}; float workload_confidence{0.0F}; std::vector keys; + std::vector sessions; }; struct PolicyContext { IoPatternSnapshot snapshot; PatternResult analysis; + std::string session_id; }; struct TraceEvent { @@ -191,6 +211,7 @@ struct EvictionCandidate { ObjectRef object; uint64_t bytes{0}; float score{0.0F}; + CacheTier target_tier{CacheTier::kL3NofSsd}; }; struct EvictionPlan { @@ -214,6 +235,13 @@ struct AdmissionResult { float confidence{0.0F}; }; +struct PolicyResult { + EvictionPlan eviction; + PrefetchPlan prefetch; + std::vector admissions; + bool degraded{false}; +}; + struct CacheViewEntry { ObjectRef object; CacheTier tier{CacheTier::kL2Segment}; diff --git a/mooncake-store/include/io_pattern/view_manager.h b/mooncake-store/include/io_pattern/view_manager.h deleted file mode 100644 index 4ef636b71d..0000000000 --- a/mooncake-store/include/io_pattern/view_manager.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "io_pattern/types.h" - -namespace mooncake::io_pattern { - -// Owns the published cache view, not the storage operations that realize it. -class CacheViewManager { - public: - virtual ~CacheViewManager() = default; - - virtual CacheView ComputeView() const = 0; - virtual void PublishEvent(const CacheEvent& event) = 0; - virtual KVMappingTable GetGlobalMapping() const = 0; -}; - -} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern_analyzer.h b/mooncake-store/include/io_pattern_analyzer.h index 7579ac39c1..9ff4932701 100644 --- a/mooncake-store/include/io_pattern_analyzer.h +++ b/mooncake-store/include/io_pattern_analyzer.h @@ -1,3 +1,4 @@ #pragma once #include "io_pattern/analyzer.h" +#include "io_pattern/threshold_analyzer.h" diff --git a/mooncake-store/include/legacy_eviction_ops.h b/mooncake-store/include/legacy_eviction_ops.h new file mode 100644 index 0000000000..8fdcc44f50 --- /dev/null +++ b/mooncake-store/include/legacy_eviction_ops.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/legacy_eviction_ops.h" diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 9b758ea7ff..11ee3fe767 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -46,6 +46,10 @@ namespace mooncake { +namespace io_pattern { +class IoPatternRuntime; +} + // Forward declaration for MasterSnapshotManager class MasterSnapshotManager; class MasterSnapshotRepository; @@ -2086,6 +2090,11 @@ class MasterService { // from any GetReplicaList caller without additional locking. std::unique_ptr promotion_sketch_; + // The IO-pattern pipeline is deliberately owned by MasterService: the + // master has the authoritative replica map and is the only component that + // can safely translate a policy plan into promotion/eviction operations. + std::unique_ptr io_pattern_runtime_; + const std::string ha_backend_type_; const std::string ha_backend_connstring_; diff --git a/mooncake-store/include/observability.h b/mooncake-store/include/observability.h new file mode 100644 index 0000000000..8e4b83cffc --- /dev/null +++ b/mooncake-store/include/observability.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/observability.h" diff --git a/mooncake-store/include/policy_strategies.h b/mooncake-store/include/policy_strategies.h new file mode 100644 index 0000000000..7a7c6001d4 --- /dev/null +++ b/mooncake-store/include/policy_strategies.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/policy_strategies.h" diff --git a/mooncake-store/include/reporter.h b/mooncake-store/include/reporter.h new file mode 100644 index 0000000000..8877123fab --- /dev/null +++ b/mooncake-store/include/reporter.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/reporter.h" diff --git a/mooncake-store/include/resilient_analyzer.h b/mooncake-store/include/resilient_analyzer.h new file mode 100644 index 0000000000..13e25d7901 --- /dev/null +++ b/mooncake-store/include/resilient_analyzer.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/resilient_analyzer.h" diff --git a/mooncake-store/include/resilient_cfm_channel.h b/mooncake-store/include/resilient_cfm_channel.h new file mode 100644 index 0000000000..654ac93cf9 --- /dev/null +++ b/mooncake-store/include/resilient_cfm_channel.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/resilient_cfm_channel.h" diff --git a/mooncake-store/include/rpc_transport.h b/mooncake-store/include/rpc_transport.h new file mode 100644 index 0000000000..b5a8f32cfc --- /dev/null +++ b/mooncake-store/include/rpc_transport.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/rpc_transport.h" diff --git a/mooncake-store/include/sliding_window_analyzer.h b/mooncake-store/include/sliding_window_analyzer.h new file mode 100644 index 0000000000..e9f03d51db --- /dev/null +++ b/mooncake-store/include/sliding_window_analyzer.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/sliding_window_analyzer.h" diff --git a/mooncake-store/include/threshold_analyzer.h b/mooncake-store/include/threshold_analyzer.h new file mode 100644 index 0000000000..a0b1ffad03 --- /dev/null +++ b/mooncake-store/include/threshold_analyzer.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/threshold_analyzer.h" diff --git a/mooncake-store/include/tier_executor.h b/mooncake-store/include/tier_executor.h new file mode 100644 index 0000000000..efac66ffdd --- /dev/null +++ b/mooncake-store/include/tier_executor.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/tier_executor.h" diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 3031dbda86..095a1b16a4 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -61,6 +61,24 @@ set(MOONCAKE_STORE_SOURCES utils/file_util.cpp task_manager.cpp local_hot_cache.cpp + io_pattern/collector_impl.cpp + io_pattern/reporter.cpp + io_pattern/cfm_client_impl.cpp + io_pattern/cfm_ingress.cpp + io_pattern/cfm_protocol.cpp + io_pattern/resilient_cfm_channel.cpp + io_pattern/feedback.cpp + io_pattern/degrading_policy_engine.cpp + io_pattern/legacy_eviction_ops.cpp + io_pattern/kmeans_analyzer.cpp + io_pattern/observability.cpp + io_pattern/sliding_window_analyzer.cpp + io_pattern/tier_executor.cpp + io_pattern/resilient_analyzer.cpp + io_pattern/rpc_transport.cpp + io_pattern/runtime.cpp + io_pattern/threshold_analyzer.cpp + io_pattern/policy_strategies.cpp ha/oplog/oplog_types.cpp ha/oplog/oplog_batch_codec.cpp ha/oplog/oplog_batch_storage.cpp diff --git a/mooncake-store/src/io_pattern/cfm_client_impl.cpp b/mooncake-store/src/io_pattern/cfm_client_impl.cpp new file mode 100644 index 0000000000..2bd502af2b --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_client_impl.cpp @@ -0,0 +1,31 @@ +#include "io_pattern/cfm_client_impl.h" + +namespace mooncake::io_pattern { + +ErrorCode CfmClientImpl::ReportSnapshot(const IoPatternSnapshot& snapshot) { + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + return channel_->SendSnapshot(snapshot) ? ErrorCode::OK + : ErrorCode::RPC_FAIL; +} + +ErrorCode CfmClientImpl::ReceivePolicy(const PolicyCommand& command) { + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + if (!policy_handler_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + return policy_handler_(command); +} + +ErrorCode CfmClientImpl::ExecutePrefetch(const PrefetchPlan& plan) { + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + return channel_->ExecutePrefetch(plan); +} + +std::optional CfmClientImpl::PollPolicy() { + return channel_ ? channel_->PollPolicy() : std::nullopt; +} + +ErrorCode CfmClientImpl::PollAndDispatchPolicy() { + const auto command = PollPolicy(); + return command ? ReceivePolicy(*command) : ErrorCode::RPC_TIMEOUT; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_ingress.cpp b/mooncake-store/src/io_pattern/cfm_ingress.cpp new file mode 100644 index 0000000000..effabd7b1a --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_ingress.cpp @@ -0,0 +1,36 @@ +#include "io_pattern/cfm_ingress.h" + +namespace mooncake::io_pattern { + +bool CfmIngress::Handle(std::string_view method, std::string_view payload) { + if (!runtime_ || !codec_) return false; + const std::string wire(payload); + if (method == "report_snapshot") { + const auto snapshot = codec_->DecodeSnapshot(wire); + if (!snapshot) return false; + runtime_->MergeSnapshot(*snapshot); + return true; + } + if (method == "report_metric_batch") { + const auto batch = codec_->DecodeMetricBatch(wire); + if (!batch) return false; + for (const auto& metric : batch->inference) { + runtime_->ReportInferenceMetrics(metric); + } + for (const auto& access : batch->accesses) { + runtime_->RecordAccess(access.object.key, access); + } + for (const auto& storage : batch->storage) { + runtime_->RecordStorageMetric(storage); + } + return true; + } + if (method == "execute_prefetch") { + const auto command = codec_->DecodePolicy(wire); + const auto* plan = command ? std::get_if(&*command) : nullptr; + return plan && runtime_->ExecuteCommand(*plan) == ErrorCode::OK; + } + return false; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_protocol.cpp b/mooncake-store/src/io_pattern/cfm_protocol.cpp new file mode 100644 index 0000000000..229025a017 --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_protocol.cpp @@ -0,0 +1,425 @@ +#include "io_pattern/cfm_protocol.h" + +#include +#include +#include + +namespace mooncake::io_pattern { +namespace { + +constexpr char kWireVersion[] = "CFM2"; +constexpr size_t kMaxWireStringBytes = 16 * 1024 * 1024; +constexpr size_t kMaxWirePayloadBytes = 64 * 1024 * 1024; +constexpr uint32_t kMaxWireRecords = 1'000'000; + +template +void Append(std::string& out, T value) { + static_assert(std::is_trivially_copyable_v); + const auto* bytes = reinterpret_cast(&value); + out.append(bytes, sizeof(value)); +} + +template +bool Read(const std::string& input, size_t& offset, T& value) { + static_assert(std::is_trivially_copyable_v); + if (input.size() - offset < sizeof(value)) return false; + std::memcpy(&value, input.data() + offset, sizeof(value)); + offset += sizeof(value); + return true; +} + +void AppendString(std::string& out, const std::string& value) { + const auto size = static_cast(value.size()); + Append(out, size); + out.append(value); +} + +bool ReadString(const std::string& input, size_t& offset, std::string& value) { + uint32_t size = 0; + if (!Read(input, offset, size) || size > kMaxWireStringBytes || + input.size() - offset < size) { + return false; + } + value.assign(input.data() + offset, size); + offset += size; + return true; +} + +void AppendObject(std::string& out, const ObjectRef& object) { + AppendString(out, object.tenant_id.value()); + AppendString(out, object.key); +} + +bool ReadObject(const std::string& input, size_t& offset, ObjectRef& object) { + std::string tenant; + if (!ReadString(input, offset, tenant) || !ReadString(input, offset, object.key)) { + return false; + } + object.tenant_id = TenantId(std::move(tenant)); + return true; +} + +template +void AppendEnum(std::string& out, Enum value) { + Append(out, static_cast(value)); +} + +template +bool ReadEnum(const std::string& input, size_t& offset, Enum& value) { + uint8_t raw = 0; + if (!Read(input, offset, raw)) return false; + value = static_cast(raw); + return true; +} + +void AppendPrefetchPlan(std::string& out, const PrefetchPlan& plan) { + AppendEnum(out, plan.strategy); + Append(out, plan.timeout_us); + Append(out, static_cast(plan.candidates.size())); + for (const auto& candidate : plan.candidates) { + AppendObject(out, candidate.object); + AppendEnum(out, candidate.source_tier); + AppendEnum(out, candidate.target_tier); + Append(out, candidate.bytes); + Append(out, candidate.priority); + Append(out, candidate.confidence); + } +} + +bool ReadPrefetchPlan(const std::string& input, size_t& offset, PrefetchPlan& plan) { + uint32_t count = 0; + if (!ReadEnum(input, offset, plan.strategy) || !Read(input, offset, plan.timeout_us) || + !Read(input, offset, count) || count > kMaxWireRecords) { + return false; + } + plan.candidates.clear(); + plan.candidates.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + PrefetchCandidate candidate; + if (!ReadObject(input, offset, candidate.object) || + !ReadEnum(input, offset, candidate.source_tier) || + !ReadEnum(input, offset, candidate.target_tier) || + !Read(input, offset, candidate.bytes) || + !Read(input, offset, candidate.priority) || + !Read(input, offset, candidate.confidence)) { + return false; + } + plan.candidates.push_back(std::move(candidate)); + } + return true; +} + +void AppendStorageMetric(std::string& out, const StorageMetric& storage) { + AppendString(out, storage.source_id); + Append(out, storage.observed_at_ns); + AppendEnum(out, storage.tier); + AppendEnum(out, storage.gc_state); + Append(out, storage.read_bandwidth_bytes_per_sec); + Append(out, storage.write_bandwidth_bytes_per_sec); + Append(out, storage.read_latency_us); + Append(out, storage.write_latency_us); + Append(out, storage.used_bytes); + Append(out, storage.capacity_bytes); + Append(out, storage.rpc_latency_us); + Append(out, storage.memory_used_ratio); +} + +bool ReadStorageMetric(const std::string& input, size_t& offset, + StorageMetric& storage) { + return ReadString(input, offset, storage.source_id) && + Read(input, offset, storage.observed_at_ns) && + ReadEnum(input, offset, storage.tier) && + ReadEnum(input, offset, storage.gc_state) && + Read(input, offset, storage.read_bandwidth_bytes_per_sec) && + Read(input, offset, storage.write_bandwidth_bytes_per_sec) && + Read(input, offset, storage.read_latency_us) && + Read(input, offset, storage.write_latency_us) && + Read(input, offset, storage.used_bytes) && + Read(input, offset, storage.capacity_bytes) && + Read(input, offset, storage.rpc_latency_us) && + Read(input, offset, storage.memory_used_ratio); +} + +void AppendHeader(std::string& out, char type) { + out.append(kWireVersion, sizeof(kWireVersion) - 1); + out.push_back(type); +} + +bool ReadHeader(const std::string& input, size_t& offset, char& type) { + if (input.size() < sizeof(kWireVersion) || input.size() > kMaxWirePayloadBytes || + input.compare(0, sizeof(kWireVersion) - 1, kWireVersion) != 0) { + return false; + } + offset = sizeof(kWireVersion) - 1; + return Read(input, offset, type); +} + +} // namespace + +std::string CfmBinaryCodec::EncodeSnapshot(const IoPatternSnapshot& snapshot) const { + std::string output; + AppendHeader(output, 'S'); + Append(output, snapshot.generated_at_ns); + Append(output, static_cast(snapshot.keys.size())); + for (const auto& key : snapshot.keys) { + AppendObject(output, key.object); + AppendString(output, key.session_id); + Append(output, key.last_access_time_ns); + Append(output, key.access_count_window); + Append(output, key.idle_time_us); + Append(output, key.block_size); + Append(output, key.transfer_eta_us); + Append(output, key.token_count); + Append(output, key.prefix_depth); + Append(output, key.prefix_fanout); + Append(output, key.match_length); + Append(output, key.continuous_prefix_length); + Append(output, key.other_replica_count); + Append(output, key.write_batch_size); + Append(output, key.write_frequency); + Append(output, key.write_object_size); + Append(output, key.recompute_cost); + Append(output, key.overwrite_ratio); + Append(output, key.replica_tiers); + AppendEnum(output, key.layout); + Append(output, key.layout_group); + Append(output, key.request_priority); + Append(output, key.active); + Append(output, key.pinned); + Append(output, key.ssd_replica_exists); + Append(output, key.write_burst); + } + Append(output, static_cast(snapshot.storage.size())); + for (const auto& storage : snapshot.storage) { + AppendStorageMetric(output, storage); + } + return output; +} + +std::string CfmBinaryCodec::EncodePrefetch(const PrefetchPlan& plan) const { + std::string output; + AppendHeader(output, 'P'); + AppendPrefetchPlan(output, plan); + return output; +} + +std::string CfmBinaryCodec::EncodeMetricBatch(const MetricBatch& batch) const { + std::string output; + AppendHeader(output, 'M'); + Append(output, static_cast(batch.inference.size())); + for (const auto& metric : batch.inference) { + AppendObject(output, metric.object); + AppendString(output, metric.session_id); + Append(output, metric.prefix_depth); + Append(output, metric.prefix_fanout); + Append(output, metric.match_length); + Append(output, metric.continuous_prefix_length); + Append(output, metric.token_count); + Append(output, metric.recompute_cost); + Append(output, metric.request_priority); + AppendEnum(output, metric.layout); + Append(output, metric.layout_group); + } + Append(output, static_cast(batch.accesses.size())); + for (const auto& access : batch.accesses) { + AppendObject(output, access.object); + Append(output, access.observed_at_ns); + Append(output, access.block_size); + Append(output, access.latency_us); + AppendEnum(output, access.tier); + AppendEnum(output, access.operation); + Append(output, access.is_hit); + Append(output, access.write_batch_size); + Append(output, access.overwrite); + } + Append(output, static_cast(batch.storage.size())); + for (const auto& storage : batch.storage) { + AppendStorageMetric(output, storage); + } + return output; +} + +std::string CfmBinaryCodec::EncodePolicy(const PolicyCommand& command) const { + std::string output; + if (const auto* eviction = std::get_if(&command)) { + AppendHeader(output, 'E'); + AppendEnum(output, eviction->source_tier); + Append(output, eviction->target_bytes); + Append(output, static_cast(eviction->candidates.size())); + for (const auto& candidate : eviction->candidates) { + AppendObject(output, candidate.object); + Append(output, candidate.bytes); + Append(output, candidate.score); + AppendEnum(output, candidate.target_tier); + } + } else if (const auto* prefetch = std::get_if(&command)) { + AppendHeader(output, 'P'); + AppendPrefetchPlan(output, *prefetch); + } else { + const auto& admission = std::get(command); + AppendHeader(output, 'A'); + AppendObject(output, admission.object); + AppendEnum(output, admission.target_tier); + AppendEnum(output, admission.decision); + Append(output, admission.confidence); + } + return output; +} + +std::optional CfmBinaryCodec::DecodePolicy( + const std::string& payload) const { + size_t offset = 0; + char type = 0; + if (!ReadHeader(payload, offset, type)) return std::nullopt; + if (type == 'P') { + PrefetchPlan plan; + return ReadPrefetchPlan(payload, offset, plan) && offset == payload.size() + ? std::optional(std::move(plan)) + : std::nullopt; + } + if (type == 'E') { + EvictionPlan plan; + uint32_t count = 0; + if (!ReadEnum(payload, offset, plan.source_tier) || + !Read(payload, offset, plan.target_bytes) || + !Read(payload, offset, count) || count > kMaxWireRecords) { + return std::nullopt; + } + plan.candidates.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + EvictionCandidate candidate; + if (!ReadObject(payload, offset, candidate.object) || + !Read(payload, offset, candidate.bytes) || + !Read(payload, offset, candidate.score) || + !ReadEnum(payload, offset, candidate.target_tier)) { + return std::nullopt; + } + plan.candidates.push_back(std::move(candidate)); + } + return offset == payload.size() + ? std::optional(std::move(plan)) + : std::nullopt; + } + if (type == 'A') { + AdmissionResult result; + if (!ReadObject(payload, offset, result.object) || + !ReadEnum(payload, offset, result.target_tier) || + !ReadEnum(payload, offset, result.decision) || + !Read(payload, offset, result.confidence) || offset != payload.size()) { + return std::nullopt; + } + return result; + } + return std::nullopt; +} + +std::optional CfmBinaryCodec::DecodeSnapshot( + const std::string& payload) const { + size_t offset = 0; + char type = 0; + IoPatternSnapshot snapshot; + uint32_t key_count = 0; + if (!ReadHeader(payload, offset, type) || type != 'S' || + !Read(payload, offset, snapshot.generated_at_ns) || + !Read(payload, offset, key_count) || key_count > kMaxWireRecords) { + return std::nullopt; + } + snapshot.keys.reserve(key_count); + for (uint32_t index = 0; index < key_count; ++index) { + KeyMetrics key; + if (!ReadObject(payload, offset, key.object) || + !ReadString(payload, offset, key.session_id) || + !Read(payload, offset, key.last_access_time_ns) || + !Read(payload, offset, key.access_count_window) || + !Read(payload, offset, key.idle_time_us) || + !Read(payload, offset, key.block_size) || + !Read(payload, offset, key.transfer_eta_us) || + !Read(payload, offset, key.token_count) || + !Read(payload, offset, key.prefix_depth) || + !Read(payload, offset, key.prefix_fanout) || + !Read(payload, offset, key.match_length) || + !Read(payload, offset, key.continuous_prefix_length) || + !Read(payload, offset, key.other_replica_count) || + !Read(payload, offset, key.write_batch_size) || + !Read(payload, offset, key.write_frequency) || + !Read(payload, offset, key.write_object_size) || + !Read(payload, offset, key.recompute_cost) || + !Read(payload, offset, key.overwrite_ratio) || + !Read(payload, offset, key.replica_tiers) || + !ReadEnum(payload, offset, key.layout) || + !Read(payload, offset, key.layout_group) || + !Read(payload, offset, key.request_priority) || + !Read(payload, offset, key.active) || !Read(payload, offset, key.pinned) || + !Read(payload, offset, key.ssd_replica_exists) || + !Read(payload, offset, key.write_burst)) { + return std::nullopt; + } + snapshot.keys.push_back(std::move(key)); + } + uint32_t storage_count = 0; + if (!Read(payload, offset, storage_count) || storage_count > kMaxWireRecords) { + return std::nullopt; + } + snapshot.storage.reserve(storage_count); + for (uint32_t index = 0; index < storage_count; ++index) { + StorageMetric storage; + if (!ReadStorageMetric(payload, offset, storage)) return std::nullopt; + snapshot.storage.push_back(std::move(storage)); + } + return offset == payload.size() ? std::optional(std::move(snapshot)) + : std::nullopt; +} + +std::optional CfmBinaryCodec::DecodeMetricBatch( + const std::string& payload) const { + size_t offset = 0; + char type = 0; + MetricBatch batch; + uint32_t count = 0; + if (!ReadHeader(payload, offset, type) || type != 'M' || + !Read(payload, offset, count) || count > kMaxWireRecords) { + return std::nullopt; + } + batch.inference.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + InferenceMetrics metric; + if (!ReadObject(payload, offset, metric.object) || + !ReadString(payload, offset, metric.session_id) || + !Read(payload, offset, metric.prefix_depth) || + !Read(payload, offset, metric.prefix_fanout) || + !Read(payload, offset, metric.match_length) || + !Read(payload, offset, metric.continuous_prefix_length) || + !Read(payload, offset, metric.token_count) || + !Read(payload, offset, metric.recompute_cost) || + !Read(payload, offset, metric.request_priority) || + !ReadEnum(payload, offset, metric.layout) || + !Read(payload, offset, metric.layout_group)) return std::nullopt; + batch.inference.push_back(std::move(metric)); + } + if (!Read(payload, offset, count) || count > kMaxWireRecords) return std::nullopt; + batch.accesses.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + AccessRecord access; + if (!ReadObject(payload, offset, access.object) || + !Read(payload, offset, access.observed_at_ns) || + !Read(payload, offset, access.block_size) || + !Read(payload, offset, access.latency_us) || + !ReadEnum(payload, offset, access.tier) || + !ReadEnum(payload, offset, access.operation) || + !Read(payload, offset, access.is_hit) || + !Read(payload, offset, access.write_batch_size) || + !Read(payload, offset, access.overwrite)) return std::nullopt; + batch.accesses.push_back(std::move(access)); + } + if (!Read(payload, offset, count) || count > kMaxWireRecords) return std::nullopt; + batch.storage.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + StorageMetric storage; + if (!ReadStorageMetric(payload, offset, storage)) return std::nullopt; + batch.storage.push_back(std::move(storage)); + } + return offset == payload.size() ? std::optional(std::move(batch)) + : std::nullopt; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/collector_impl.cpp b/mooncake-store/src/io_pattern/collector_impl.cpp new file mode 100644 index 0000000000..e31b06c223 --- /dev/null +++ b/mooncake-store/src/io_pattern/collector_impl.cpp @@ -0,0 +1,183 @@ +#include "io_pattern/collector_impl.h" + +#include +#include +#include + +namespace mooncake::io_pattern { +namespace { +uint64_t NowNs() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} +} + +void IoPatternCollectorImpl::ReportInferenceMetrics( + const InferenceMetrics& metrics) { + std::lock_guard lock(mutex_); + if (reporter_ && !reporter_->Enqueue(metrics)) ++dropped_; + if (!key_metrics_.contains(metrics.object) && config_.max_total_keys != 0 && + key_metrics_.size() >= config_.max_total_keys) { + degraded_ = true; + ++dropped_; + return; + } + if (!key_metrics_.contains(metrics.object) && + config_.max_keys_per_tenant != 0 && + tenant_key_counts_[metrics.object.tenant_id] >= + config_.max_keys_per_tenant) { + ++dropped_; + return; + } + if (!key_metrics_.contains(metrics.object)) + ++tenant_key_counts_[metrics.object.tenant_id]; + auto& value = key_metrics_[metrics.object]; + value.object = metrics.object; + value.session_id = metrics.session_id; + value.layout = metrics.layout; + value.layout_group = metrics.layout_group; + value.prefix_depth = metrics.prefix_depth; + value.prefix_fanout = metrics.prefix_fanout; + value.match_length = metrics.match_length; + value.continuous_prefix_length = metrics.continuous_prefix_length; + value.token_count = metrics.token_count; + value.recompute_cost = metrics.recompute_cost; + value.request_priority = metrics.request_priority; +} + +void IoPatternCollectorImpl::RecordAccess(const std::string& key, + const AccessRecord& record) { + std::lock_guard lock(mutex_); + if (reporter_ && !reporter_->EnqueueAccess(record)) ++dropped_; + ObjectRef object = record.object; + if (!key.empty()) object.key = key; + if (!key_metrics_.contains(object) && config_.max_total_keys != 0 && + key_metrics_.size() >= config_.max_total_keys) { + degraded_ = true; + ++dropped_; + return; + } + if (!key_metrics_.contains(object) && config_.max_keys_per_tenant != 0 && + tenant_key_counts_[object.tenant_id] >= config_.max_keys_per_tenant) { + ++dropped_; + return; + } + if (!key_metrics_.contains(object)) ++tenant_key_counts_[object.tenant_id]; + auto& value = key_metrics_[object]; + value.object = object; + ++value.access_count_window; + value.last_access_time_ns = + std::max(value.last_access_time_ns, record.observed_at_ns); + value.block_size = std::max(value.block_size, record.block_size); + value.replica_tiers |= CacheTierBit(record.tier); + value.active = value.active || record.is_hit; + if (record.operation == IoOperation::kPut) { + ++write_counts_[object]; + if (record.overwrite) ++overwrite_counts_[object]; + value.write_frequency = + static_cast(std::min(write_counts_[object], + UINT32_MAX)); + value.write_batch_size = + std::max(value.write_batch_size, record.write_batch_size); + value.write_object_size = std::max(value.write_object_size, + record.block_size); + value.overwrite_ratio = + static_cast(overwrite_counts_[object]) / + static_cast(write_counts_[object]); + value.write_burst = record.write_batch_size >= 16; + } +} + +void IoPatternCollectorImpl::RecordStorageMetric(const StorageMetric& metric) { + std::lock_guard lock(mutex_); + if (reporter_ && !reporter_->EnqueueStorage(metric)) ++dropped_; + StorageMetricKey key{metric.source_id, metric.tier}; + auto it = storage_metrics_.find(key); + if (it == storage_metrics_.end() || + metric.observed_at_ns >= it->second.observed_at_ns) { + storage_metrics_[std::move(key)] = metric; + } +} + +void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { + std::lock_guard lock(mutex_); + for (const auto& metrics : snapshot.keys) { + if (!key_metrics_.contains(metrics.object) && + config_.max_total_keys != 0 && + key_metrics_.size() >= config_.max_total_keys) { + degraded_ = true; + ++dropped_; + continue; + } + if (!key_metrics_.contains(metrics.object) && + config_.max_keys_per_tenant != 0 && + tenant_key_counts_[metrics.object.tenant_id] >= + config_.max_keys_per_tenant) { + ++dropped_; + continue; + } + if (!key_metrics_.contains(metrics.object)) { + ++tenant_key_counts_[metrics.object.tenant_id]; + } + key_metrics_[metrics.object] = metrics; + } + for (const auto& metric : snapshot.storage) { + StorageMetricKey key{metric.source_id, metric.tier}; + auto it = storage_metrics_.find(key); + if (it == storage_metrics_.end() || + metric.observed_at_ns >= it->second.observed_at_ns) { + storage_metrics_[std::move(key)] = metric; + } + } +} + +IoPatternSnapshot IoPatternCollectorImpl::GetSnapshot() const { + std::lock_guard lock(mutex_); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = NowNs(); + snapshot.keys.reserve(key_metrics_.size()); + for (const auto& [object, metrics] : key_metrics_) { + (void)object; + auto copy = metrics; + if (copy.last_access_time_ns != 0 && + snapshot.generated_at_ns > copy.last_access_time_ns) { + copy.idle_time_us = + (snapshot.generated_at_ns - copy.last_access_time_ns) / 1000; + } + snapshot.keys.push_back(std::move(copy)); + } + snapshot.storage.reserve(storage_metrics_.size()); + for (const auto& [key, metric] : storage_metrics_) { + (void)key; + snapshot.storage.push_back(metric); + } + std::sort(snapshot.keys.begin(), snapshot.keys.end(), + [](const KeyMetrics& a, const KeyMetrics& b) { + if (a.object.tenant_id != b.object.tenant_id) + return a.object.tenant_id < b.object.tenant_id; + return a.object.key < b.object.key; + }); + return snapshot; +} + +uint64_t IoPatternCollectorImpl::dropped() const { + std::lock_guard lock(mutex_); + return dropped_; +} + +bool IoPatternCollectorImpl::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +bool IoPatternCollectorImpl::FlushReports() { + std::shared_ptr reporter; + { + std::lock_guard lock(mutex_); + reporter = reporter_; + } + return !reporter || reporter->Flush(); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/degrading_policy_engine.cpp b/mooncake-store/src/io_pattern/degrading_policy_engine.cpp new file mode 100644 index 0000000000..bdcbe6d06f --- /dev/null +++ b/mooncake-store/src/io_pattern/degrading_policy_engine.cpp @@ -0,0 +1,60 @@ +#include "io_pattern/degrading_policy_engine.h" + +namespace mooncake::io_pattern { + +void DegradingPolicyEngine::RecordFailure() { + std::lock_guard lock(mutex_); + ++consecutive_failures_; + if (failure_threshold_ != 0 && consecutive_failures_ >= failure_threshold_) + degraded_ = true; +} + +void DegradingPolicyEngine::RecordSuccess() { + std::lock_guard lock(mutex_); + consecutive_failures_ = 0; +} + +void DegradingPolicyEngine::ForceDegraded(bool value) { + std::lock_guard lock(mutex_); + degraded_ = value; + if (!value) consecutive_failures_ = 0; +} + +bool DegradingPolicyEngine::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +size_t DegradingPolicyEngine::consecutive_failures() const { + std::lock_guard lock(mutex_); + return consecutive_failures_; +} + +std::shared_ptr DegradingPolicyEngine::Active() const { + std::lock_guard lock(mutex_); + return degraded_ ? fallback_ : primary_; +} + +EvictionPlan DegradingPolicyEngine::PlanEviction(const PolicyContext& context, + CacheTier tier, + uint64_t bytes) const { + auto engine = Active(); + return engine ? engine->PlanEviction(context, tier, bytes) + : EvictionPlan{.source_tier = tier, .target_bytes = bytes}; +} + +PrefetchPlan DegradingPolicyEngine::PlanPrefetch( + const PolicyContext& context, const TraceHistory& trace) const { + auto engine = Active(); + return engine ? engine->PlanPrefetch(context, trace) : PrefetchPlan{}; +} + +AdmissionResult DegradingPolicyEngine::DecideAdmission( + const ObjectRef& object, CacheTier tier, + const PolicyContext& context) const { + auto engine = Active(); + return engine ? engine->DecideAdmission(object, tier, context) + : AdmissionResult{.object = object, .target_tier = tier}; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/feedback.cpp b/mooncake-store/src/io_pattern/feedback.cpp new file mode 100644 index 0000000000..7598dbc99b --- /dev/null +++ b/mooncake-store/src/io_pattern/feedback.cpp @@ -0,0 +1,58 @@ +#include "io_pattern/feedback.h" + +namespace mooncake::io_pattern { + +void PolicyFeedbackWindow::Record(PolicyFeedbackSample sample) { + std::lock_guard lock(mutex_); + if (capacity_ == 0) return; + if (samples_.size() == capacity_) samples_.pop_front(); + samples_.push_back(sample); +} + +PolicyFeedbackStats PolicyFeedbackWindow::Snapshot() const { + std::lock_guard lock(mutex_); + PolicyFeedbackStats stats; + stats.samples = samples_.size(); + for (const auto& sample : samples_) { + stats.hit_rate_delta += sample.hit_rate_delta; + stats.eviction_churn += sample.eviction_churn; + stats.ttft_delta += sample.ttft_delta; + stats.prefetch_accuracy += sample.prefetch_accuracy; + } + if (stats.samples != 0) { + const float divisor = static_cast(stats.samples); + stats.hit_rate_delta /= divisor; + stats.eviction_churn /= divisor; + stats.ttft_delta /= divisor; + stats.prefetch_accuracy /= divisor; + } + return stats; +} + +bool AdaptivePolicyTuner::Tune(const PolicyFeedbackStats& stats, + ScoreBasedEvictionConfig& config) { + if (stats.hit_rate_delta < 0.0F) { + ++negative_streak_; + } else { + negative_streak_ = 0; + } + if (negative_windows_ == 0 || negative_streak_ < negative_windows_) { + if (stats.eviction_churn > 0.5F || stats.prefetch_accuracy < 0.2F || + stats.ttft_delta > 0.1F) { + conservative_ = true; + config.prefix_weight *= 0.9F; + config.recompute_weight *= 0.9F; + if (persistence_) persistence_(config); + return true; + } + return false; + } + config.frequency_weight *= 0.8F; + config.idle_weight *= 1.1F; + conservative_ = true; + negative_streak_ = 0; + if (persistence_) persistence_(config); + return true; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/kmeans_analyzer.cpp b/mooncake-store/src/io_pattern/kmeans_analyzer.cpp new file mode 100644 index 0000000000..971e1a92a3 --- /dev/null +++ b/mooncake-store/src/io_pattern/kmeans_analyzer.cpp @@ -0,0 +1,161 @@ +#include "io_pattern/kmeans_analyzer.h" + +#include +#include +#include +#include + +namespace mooncake::io_pattern { +namespace { + +// Token length, fanout, block size, access frequency and prefix-match length +// are the five documented dimensions. They are normalized against the active +// sliding window before distance calculation. +using Feature = std::array; + +struct SessionFeatures { + Feature values{}; + size_t samples{0}; +}; + +Feature ToFeature(const KeyMetrics& key) { + return {static_cast(key.token_count), + static_cast(key.prefix_fanout), + static_cast(key.block_size), + static_cast(key.access_count_window), + static_cast(key.match_length)}; +} + +float Distance(const Feature& lhs, const Feature& rhs, const Feature& scale) { + float distance = 0.0F; + for (size_t index = 0; index < lhs.size(); ++index) { + const float normalized = (lhs[index] - rhs[index]) / + std::max(1.0F, scale[index]); + distance += normalized * normalized; + } + return distance; +} + +WorkloadType Classify(const Feature& feature, + const ThresholdAnalyzerConfig& config) { + if (feature[0] > config.code_agent_token_count && + feature[1] > config.code_agent_prefix_fanout && + feature[4] > config.code_agent_match_length) { + return WorkloadType::kCodeAgent; + } + if (feature[2] < config.recommendation_block_size && + feature[3] > config.recommendation_frequency) { + return WorkloadType::kGenerativeRecommendation; + } + if (feature[1] > config.conversation_prefix_fanout && + feature[4] > config.conversation_match_length) { + return WorkloadType::kMultiTurnConversation; + } + return WorkloadType::kMixed; +} + +} // namespace + +PatternResult KMeansWorkloadAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + PatternResult result; + if (snapshot.keys.empty()) { + result.workload_type = WorkloadType::kMixed; + return result; + } + + std::unordered_map by_session; + Feature scale{1.0F, 1.0F, 1.0F, 1.0F}; + for (const auto& key : snapshot.keys) { + const std::string session = key.session_id.empty() + ? key.object.tenant_id.value() + ":" + key.object.key + : key.session_id; + auto& aggregate = by_session[session]; + const auto feature = ToFeature(key); + for (size_t index = 0; index < feature.size(); ++index) { + aggregate.values[index] += feature[index]; + scale[index] = std::max(scale[index], feature[index]); + } + ++aggregate.samples; + } + + std::vector session_ids; + std::vector samples; + session_ids.reserve(by_session.size()); + samples.reserve(by_session.size()); + for (auto& [session, aggregate] : by_session) { + for (auto& value : aggregate.values) { + value /= static_cast(aggregate.samples); + } + session_ids.push_back(session); + samples.push_back(aggregate.values); + } + + const size_t cluster_count = std::min(3, samples.size()); + std::vector centroids(samples.begin(), samples.begin() + cluster_count); + std::vector assignments(samples.size(), 0); + for (uint32_t iteration = 0; iteration < config_.iterations; ++iteration) { + std::vector sums(cluster_count); + std::vector counts(cluster_count, 0); + for (size_t sample_index = 0; sample_index < samples.size(); ++sample_index) { + size_t best = 0; + float best_distance = Distance(samples[sample_index], centroids[0], scale); + for (size_t cluster = 1; cluster < cluster_count; ++cluster) { + const float distance = Distance(samples[sample_index], centroids[cluster], scale); + if (distance < best_distance) { + best = cluster; + best_distance = distance; + } + } + assignments[sample_index] = best; + ++counts[best]; + for (size_t field = 0; field < samples[sample_index].size(); ++field) { + sums[best][field] += samples[sample_index][field]; + } + } + for (size_t cluster = 0; cluster < cluster_count; ++cluster) { + if (counts[cluster] == 0) continue; + for (size_t field = 0; field < centroids[cluster].size(); ++field) { + centroids[cluster][field] = sums[cluster][field] / + static_cast(counts[cluster]); + } + } + } + + std::vector labels; + labels.reserve(cluster_count); + for (const auto& centroid : centroids) labels.push_back(Classify(centroid, config_.thresholds)); + WorkloadType global = labels[assignments.front()]; + bool mixed = false; + for (size_t index = 0; index < samples.size(); ++index) { + const auto type = labels[assignments[index]]; + result.sessions.push_back( + SessionPattern{.session_id = session_ids[index], .workload_type = type, + .confidence = 1.0F / (1.0F + Distance( + samples[index], centroids[assignments[index]], scale))}); + if (type != global) mixed = true; + } + result.workload_type = mixed ? WorkloadType::kMixed : global; + result.workload_confidence = mixed ? 0.5F : result.sessions.front().confidence; + + ThresholdAnalyzer key_analyzer(config_.thresholds); + result.keys = key_analyzer.Analyze(snapshot).keys; + return result; +} + +WorkloadType KMeansWorkloadAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + return Analyze(snapshot).workload_type; +} + +float KMeansWorkloadAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto result = Analyze(snapshot); + const auto it = std::find_if(result.keys.begin(), result.keys.end(), + [&object](const KeyPattern& key) { + return key.object == object; + }); + return it == result.keys.end() ? 0.0F : it->confidence; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/legacy_eviction_ops.cpp b/mooncake-store/src/io_pattern/legacy_eviction_ops.cpp new file mode 100644 index 0000000000..ea35682864 --- /dev/null +++ b/mooncake-store/src/io_pattern/legacy_eviction_ops.cpp @@ -0,0 +1,35 @@ +#include "io_pattern/legacy_eviction_ops.h" + +#include + +namespace mooncake::io_pattern { + +EvictionPlan LegacyEvictionOps::Evaluate(const PolicyContext& context, + CacheTier tier, + uint64_t target_bytes) const { + EvictionPlan plan{.source_tier = tier, .target_bytes = target_bytes}; + if (!strategy_ || target_bytes == 0) return plan; + std::lock_guard lock(mutex_); + for (const auto& key : context.snapshot.keys) { + if (key.replica_tiers & CacheTierBit(tier)) { + strategy_->AddKey(key.object.tenant_id.MakeScopedKey(key.object.key)); + } + } + uint64_t bytes = 0; + while (bytes < target_bytes) { + const auto scoped = strategy_->EvictKey(); + if (scoped.empty()) break; + auto [tenant, key] = TenantId::ParseScopedKey(scoped); + auto it = std::find_if(context.snapshot.keys.begin(), context.snapshot.keys.end(), + [&](const KeyMetrics& value) { + return value.object.tenant_id == tenant && + value.object.key == key; + }); + if (it == context.snapshot.keys.end()) continue; + plan.candidates.push_back({it->object, it->block_size, 0.0F}); + bytes += it->block_size; + } + return plan; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/observability.cpp b/mooncake-store/src/io_pattern/observability.cpp new file mode 100644 index 0000000000..bb920fb786 --- /dev/null +++ b/mooncake-store/src/io_pattern/observability.cpp @@ -0,0 +1,63 @@ +#include "io_pattern/observability.h" + +#include + +namespace mooncake::io_pattern { + +void IoPatternObservability::RecordCollectLatency(uint64_t value) { + std::lock_guard lock(mutex_); + values_.collect_latency_us = std::max(values_.collect_latency_us, value); +} + +void IoPatternObservability::RecordAnalyzeLatency(uint64_t value) { + std::lock_guard lock(mutex_); + values_.analyze_latency_us = std::max(values_.analyze_latency_us, value); +} + +void IoPatternObservability::RecordPolicyDecision(bool strategy_hit) { + std::lock_guard lock(mutex_); + ++values_.policy_decisions; + ++values_.strategy_trials; + if (strategy_hit) ++values_.strategy_hits; +} + +void IoPatternObservability::RecordFalsePositive() { + std::lock_guard lock(mutex_); + ++values_.false_positives; +} + +void IoPatternObservability::RecordDegrade() { + std::lock_guard lock(mutex_); + ++values_.degrade_count; +} + +void IoPatternObservability::RecordReportDrop(uint64_t count) { + std::lock_guard lock(mutex_); + values_.report_drop_count += count; +} + +IoPatternObservabilitySnapshot IoPatternObservability::Snapshot() const { + std::lock_guard lock(mutex_); + auto result = values_; + result.strategy_hit_rate = result.strategy_trials == 0 + ? 0.0F + : static_cast(result.strategy_hits) / + static_cast(result.strategy_trials); + result.false_positive_rate = result.strategy_trials == 0 + ? 0.0F + : static_cast(result.false_positives) / + static_cast(result.strategy_trials); + return result; +} + +IoPatternObservabilitySnapshot IoPatternObservability::Snapshot( + double window_seconds) const { + auto result = Snapshot(); + if (window_seconds > 0.0) { + result.policy_decision_qps = + static_cast(result.policy_decisions / window_seconds); + } + return result; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp new file mode 100644 index 0000000000..98e00f308c --- /dev/null +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -0,0 +1,213 @@ +#include "io_pattern/policy_strategies.h" + +#include +#include + +namespace mooncake::io_pattern { +namespace { + +const KeyPattern* FindPattern(const ObjectRef& object, + const PatternResult& result) { + const auto it = std::find_if( + result.keys.begin(), result.keys.end(), + [&object](const KeyPattern& pattern) { return pattern.object == object; }); + return it == result.keys.end() ? nullptr : &*it; +} + +const KeyMetrics* FindMetrics(const ObjectRef& object, + const IoPatternSnapshot& snapshot) { + const auto it = std::find_if( + snapshot.keys.begin(), snapshot.keys.end(), + [&object](const KeyMetrics& key) { return key.object == object; }); + return it == snapshot.keys.end() ? nullptr : &*it; +} + +bool HasLowerTierReplica(const KeyMetrics& key, CacheTier tier) { + const auto tier_index = static_cast(tier); + for (uint8_t index = tier_index + 1; + index <= static_cast(CacheTier::kL3NofSsd); ++index) { + if (key.replica_tiers & static_cast(1U << index)) { + return true; + } + } + return false; +} + +CacheTier TierDownTarget(CacheTier source, TierDownMode mode) { + if (source == CacheTier::kL3NofSsd) return CacheTier::kL3NofSsd; + if (mode == TierDownMode::kSkipHost && source == CacheTier::kL0Hbm) { + return CacheTier::kL2Segment; + } + // Prefix-affinity keeps the immediate next tier as the placement target; + // callers may co-locate grouped prefixes within that tier. + return static_cast(static_cast(source) + 1); +} + +} // namespace + +EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, + CacheTier tier, + uint64_t target_bytes) const { + EvictionPlan plan{.source_tier = tier, .target_bytes = target_bytes}; + uint64_t max_block_size = 0; + uint32_t max_other_replicas = 0; + for (const auto& key : context.snapshot.keys) { + if ((key.replica_tiers & CacheTierBit(tier)) == 0 || key.pinned) { + continue; + } + max_block_size = std::max(max_block_size, key.block_size); + max_other_replicas = std::max(max_other_replicas, + key.other_replica_count); + } + for (const auto& key : context.snapshot.keys) { + if ((key.replica_tiers & CacheTierBit(tier)) == 0 || key.pinned) { + continue; + } + const auto* pattern = FindPattern(key.object, context.analysis); + if (pattern == nullptr) { + continue; + } + const float normalized_block = max_block_size == 0 + ? 0.0F + : static_cast(key.block_size) / + static_cast(max_block_size); + const float normalized_other_replicas = + max_other_replicas == 0 + ? 0.0F + : static_cast(key.other_replica_count) / + static_cast(max_other_replicas); + float score = 0.0F; + switch (tier) { + case CacheTier::kL0Hbm: + score = config_.idle_weight * pattern->idle_score - + config_.frequency_weight * pattern->frequency_score - + config_.prefix_weight * pattern->prefix_score - + config_.recompute_weight * pattern->recompute_score; + break; + case CacheTier::kL1Host: + case CacheTier::kL2Segment: + score = config_.idle_weight * pattern->idle_score - + config_.frequency_weight * pattern->frequency_score + + config_.lower_replica_weight * + (HasLowerTierReplica(key, tier) ? 1.0F : 0.0F) - + config_.prefix_weight * pattern->prefix_score - + config_.recompute_weight * pattern->recompute_score; + break; + case CacheTier::kL3NofSsd: + score = config_.idle_weight * pattern->idle_score * + normalized_block - + config_.frequency_weight * pattern->frequency_score - + config_.recompute_weight * pattern->recompute_score + + config_.other_replica_weight * normalized_other_replicas; + break; + } + plan.candidates.push_back( + EvictionCandidate{.object = key.object, + .bytes = key.block_size, + .score = score, + .target_tier = TierDownTarget( + tier, config_.tier_down_mode)}); + } + std::sort(plan.candidates.begin(), plan.candidates.end(), + [](const EvictionCandidate& lhs, const EvictionCandidate& rhs) { + return lhs.score > rhs.score; + }); + if (target_bytes == 0) { + plan.candidates.clear(); + return plan; + } + if (config_.max_candidates != 0 && + plan.candidates.size() > config_.max_candidates) { + plan.candidates.resize(config_.max_candidates); + } + uint64_t selected_bytes = 0; + auto end = plan.candidates.begin(); + while (end != plan.candidates.end()) { + if (end->bytes > target_bytes - selected_bytes) { + break; + } + selected_bytes += end->bytes; + ++end; + } + plan.candidates.erase(end, plan.candidates.end()); + return plan; +} + +AdmissionResult PrefixMatchAdmissionOps::Evaluate( + const ObjectRef& object, CacheTier target_tier, + const PolicyContext& context) const { + AdmissionResult result{.object = object, .target_tier = target_tier}; + const auto* key = FindMetrics(object, context.snapshot); + if (key == nullptr) { + return result; + } + if (target_tier == CacheTier::kL0Hbm) { + result.decision = key->match_length >= config_.hbm_match_length + ? AdmissionDecision::kAdmit + : AdmissionDecision::kRejectPrefix; + const auto threshold = std::max(1U, config_.hbm_match_length); + result.confidence = key->match_length == 0 + ? 0.0F + : std::min(1.0F, static_cast( + key->match_length) / + threshold); + return result; + } + result.decision = key->access_count_window >= config_.frequency_threshold + ? AdmissionDecision::kAdmit + : AdmissionDecision::kRejectFrequency; + if (result.decision == AdmissionDecision::kAdmit && + !context.snapshot.storage.empty() && + context.snapshot.storage.front().memory_used_ratio >= + config_.max_memory_used_ratio) { + result.decision = AdmissionDecision::kRejectWatermark; + } + result.confidence = + result.decision == AdmissionDecision::kAdmit ? 1.0F : 0.0F; + return result; +} + +PrefetchPlan TraceBasedPrefetchOps::Evaluate( + const PolicyContext& context, const TraceHistory& trace) const { + PrefetchPlan plan{.strategy = config_.strategy, + .timeout_us = config_.timeout_us}; + std::unordered_map seen; + for (const auto& event : trace.events) { + if (!event.is_hit || + event.match_length <= config_.match_length_threshold || + seen.contains(event.object)) { + continue; + } + const auto* key = FindMetrics(event.object, context.snapshot); + if (key == nullptr) { + continue; + } + PrefetchCandidate candidate; + candidate.object = event.object; + candidate.bytes = key->block_size; + const auto* pattern = FindPattern(event.object, context.analysis); + candidate.confidence = pattern == nullptr ? 0.0F : pattern->confidence; + if (candidate.confidence < config_.minimum_confidence) { + continue; + } + if (key->replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) { + candidate.source_tier = CacheTier::kL3NofSsd; + candidate.target_tier = CacheTier::kL2Segment; + } else if (key->replica_tiers & CacheTierBit(CacheTier::kL2Segment)) { + candidate.source_tier = CacheTier::kL2Segment; + candidate.target_tier = CacheTier::kL1Host; + } else { + continue; + } + candidate.priority = static_cast(event.match_length); + plan.candidates.push_back(candidate); + seen.emplace(event.object, true); + if (config_.max_candidates != 0 && + plan.candidates.size() >= config_.max_candidates) { + break; + } + } + return plan; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/reporter.cpp b/mooncake-store/src/io_pattern/reporter.cpp new file mode 100644 index 0000000000..c8bf7f4fe8 --- /dev/null +++ b/mooncake-store/src/io_pattern/reporter.cpp @@ -0,0 +1,143 @@ +#include "io_pattern/reporter.h" + +#include + +namespace mooncake::io_pattern { + +IoPatternReporter::IoPatternReporter(size_t capacity, MetricBatchSink sink, + size_t per_tenant_capacity) + : capacity_(capacity), + sink_(std::move(sink)), + per_tenant_capacity_(per_tenant_capacity) {} + +IoPatternReporter::~IoPatternReporter() { Stop(); } + +void IoPatternReporter::Start() { + std::lock_guard lock(mutex_); + if (running_) return; + running_ = true; + worker_ = std::thread([this] { + std::unique_lock lock(mutex_); + while (running_) { + const size_t size = batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); + const auto interval = + (capacity_ == 0 || size * 2 >= capacity_) + ? std::chrono::milliseconds(100) + : (size == 0 ? std::chrono::milliseconds(1000) + : std::chrono::milliseconds(500)); + condition_.wait_for(lock, interval, [this] { return !running_; }); + if (!running_) break; + lock.unlock(); + Flush(); + lock.lock(); + } + }); +} + +void IoPatternReporter::Stop() { + { + std::lock_guard lock(mutex_); + if (!running_) return; + running_ = false; + } + condition_.notify_all(); + if (worker_.joinable()) worker_.join(); + Flush(); +} + +bool IoPatternReporter::Enqueue(InferenceMetrics metrics) { + const auto tenant = metrics.object.tenant_id; + return EnqueueImpl( + [value = std::move(metrics)](MetricBatch& batch) { + batch.inference.push_back(value); + }, + tenant); +} + +bool IoPatternReporter::EnqueueAccess(AccessRecord record) { + const auto tenant = record.object.tenant_id; + return EnqueueImpl( + [value = std::move(record)](MetricBatch& batch) { + batch.accesses.push_back(value); + }, + tenant); +} + +bool IoPatternReporter::EnqueueStorage(StorageMetric metric) { + return EnqueueImpl([value = std::move(metric)](MetricBatch& batch) { + batch.storage.push_back(value); + }, TenantId::Default()); +} + +bool IoPatternReporter::EnqueueImpl(std::function append, + const TenantId& tenant) { + std::lock_guard lock(mutex_); + const size_t size = batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); + if (!sink_ || size >= capacity_) { + ++dropped_; + return false; + } + if (per_tenant_capacity_ != 0 && + tenant_pending_[tenant] >= per_tenant_capacity_) { + ++dropped_; + return false; + } + append(batch_); + ++tenant_pending_[tenant]; + condition_.notify_one(); + return true; +} + +bool IoPatternReporter::Flush() { + MetricBatch outgoing; + { + std::lock_guard lock(mutex_); + if (batch_.inference.empty() && batch_.accesses.empty() && + batch_.storage.empty()) { + return true; + } + outgoing = std::move(batch_); + batch_ = {}; + tenant_pending_.clear(); + } + if (!sink_(outgoing)) { + std::lock_guard lock(mutex_); + ++dropped_; + return false; + } + std::lock_guard lock(mutex_); + reported_ += outgoing.inference.size() + outgoing.accesses.size() + + outgoing.storage.size(); + return true; +} + +size_t IoPatternReporter::pending() const { + std::lock_guard lock(mutex_); + return batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); +} + +uint64_t IoPatternReporter::dropped() const { + std::lock_guard lock(mutex_); + return dropped_; +} + +uint64_t IoPatternReporter::reported() const { + std::lock_guard lock(mutex_); + return reported_; +} + +std::chrono::milliseconds IoPatternReporter::RecommendedFlushInterval() const { + std::lock_guard lock(mutex_); + const size_t size = batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); + if (capacity_ == 0 || size * 2 >= capacity_) { + return std::chrono::milliseconds(100); + } + if (size == 0) return std::chrono::milliseconds(1000); + return std::chrono::milliseconds(500); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_analyzer.cpp b/mooncake-store/src/io_pattern/resilient_analyzer.cpp new file mode 100644 index 0000000000..b850a37398 --- /dev/null +++ b/mooncake-store/src/io_pattern/resilient_analyzer.cpp @@ -0,0 +1,67 @@ +#include "io_pattern/resilient_analyzer.h" + +namespace mooncake::io_pattern { + +PatternResult ResilientAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + if (!primary_) return Fallback(); + try { + auto result = primary_->Analyze(snapshot); + RecordSuccess(result); + return result; + } catch (...) { + RecordFailure(); + return Fallback(); + } +} + +WorkloadType ResilientAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + return Analyze(snapshot).workload_type; +} + +float ResilientAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto result = Analyze(snapshot); + for (const auto& key : result.keys) { + if (key.object == object) return key.confidence; + } + return 0.0F; +} + +void ResilientAnalyzer::RecordFailure() const { + std::lock_guard lock(mutex_); + ++failures_; + if (failure_threshold_ != 0 && failures_ >= failure_threshold_) + degraded_ = true; +} + +void ResilientAnalyzer::RecordSuccess(const PatternResult& result) const { + std::lock_guard lock(mutex_); + last_result_ = result; + failures_ = 0; + degraded_ = false; +} + +PatternResult ResilientAnalyzer::Fallback() const { + std::lock_guard lock(mutex_); + if (!last_result_.keys.empty() || last_result_.workload_type != WorkloadType::kUnknown) + return last_result_; + PatternResult result; + result.workload_type = WorkloadType::kMixed; + return result; +} + +bool ResilientAnalyzer::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +size_t ResilientAnalyzer::failures() const { + std::lock_guard lock(mutex_); + return failures_; +} + +PatternResult ResilientAnalyzer::FallbackResult() const { return Fallback(); } + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp new file mode 100644 index 0000000000..4786cdad53 --- /dev/null +++ b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp @@ -0,0 +1,80 @@ +#include "io_pattern/resilient_cfm_channel.h" + +namespace mooncake::io_pattern { + +template +bool ResilientCfmChannel::Retry(Operation&& operation) { + if (!delegate_) { + RecordFailure(); + return false; + } + for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { + if (operation()) { + RecordSuccess(); + return true; + } + } + RecordFailure(); + return false; +} + +bool ResilientCfmChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { + return Retry([&] { return delegate_->SendSnapshot(snapshot); }); +} + +std::optional ResilientCfmChannel::PollPolicy() { + if (!delegate_) { + RecordFailure(); + return std::nullopt; + } + for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { + auto result = delegate_->PollPolicy(); + if (result.has_value()) { + RecordSuccess(); + return result; + } + } + RecordFailure(); + return std::nullopt; +} + +ErrorCode ResilientCfmChannel::ExecutePrefetch(const PrefetchPlan& plan) { + ErrorCode result = ErrorCode::RPC_FAIL; + if (!delegate_) { + RecordFailure(); + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { + result = delegate_->ExecutePrefetch(plan); + if (result == ErrorCode::OK) { + RecordSuccess(); + return result; + } + } + RecordFailure(); + return result; +} + +void ResilientCfmChannel::RecordSuccess() { + std::lock_guard lock(mutex_); + consecutive_failures_ = 0; + degraded_ = false; +} + +void ResilientCfmChannel::RecordFailure() { + std::lock_guard lock(mutex_); + ++consecutive_failures_; + if (consecutive_failures_ >= config_.degrade_after_failures) degraded_ = true; +} + +bool ResilientCfmChannel::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +uint64_t ResilientCfmChannel::consecutive_failures() const { + std::lock_guard lock(mutex_); + return consecutive_failures_; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/rpc_transport.cpp b/mooncake-store/src/io_pattern/rpc_transport.cpp new file mode 100644 index 0000000000..429d894bae --- /dev/null +++ b/mooncake-store/src/io_pattern/rpc_transport.cpp @@ -0,0 +1,111 @@ +#include "io_pattern/rpc_transport.h" + +namespace mooncake::io_pattern { + +bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { + std::lock_guard lock(mutex_); + authenticated_ = token == auth_token_; + return authenticated_; +} + +bool InProcessCfmRpcTransport::Send(std::string_view method, + std::string_view payload, + std::chrono::milliseconds) { + std::lock_guard lock(mutex_); + if (!authenticated_) return false; + return !send_handler_ || send_handler_(method, payload); +} + +std::optional InProcessCfmRpcTransport::Receive( + std::string_view method, std::chrono::milliseconds) { + std::lock_guard lock(mutex_); + if (!authenticated_ || method != "poll_policy" || policies_.empty()) { + return std::nullopt; + } + auto payload = std::move(policies_.front()); + policies_.pop(); + return payload; +} + +void InProcessCfmRpcTransport::EnqueuePolicy(std::string payload) { + std::lock_guard lock(mutex_); + policies_.push(std::move(payload)); +} + +void InProcessCfmRpcTransport::SetSendHandler(SendHandler handler) { + std::lock_guard lock(mutex_); + send_handler_ = std::move(handler); +} + +bool CfmRpcChannel::EnsureAuthenticated() { + std::lock_guard lock(authentication_mutex_); + if (authenticated_) return true; + authenticated_ = transport_ && transport_->Authenticate(config_.auth_token); + return authenticated_; +} + +bool CfmRpcChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { + if (!transport_ || !codec_ || !EnsureAuthenticated()) return false; + return transport_->Send("report_snapshot", codec_->EncodeSnapshot(snapshot), + config_.timeout); +} + +std::optional CfmRpcChannel::PollPolicy() { + if (!transport_ || !codec_ || !EnsureAuthenticated()) return std::nullopt; + const auto payload = transport_->Receive("poll_policy", config_.timeout); + return payload ? codec_->DecodePolicy(*payload) : std::nullopt; +} + +ErrorCode CfmRpcChannel::ExecutePrefetch(const PrefetchPlan& plan) { + if (!transport_ || !codec_ || !EnsureAuthenticated()) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + return transport_->Send("execute_prefetch", codec_->EncodePrefetch(plan), + config_.timeout) + ? ErrorCode::OK + : ErrorCode::RPC_TIMEOUT; +} + +bool CfmRpcChannel::SendMetricBatch(const MetricBatch& batch) { + if (!transport_ || !codec_ || !EnsureAuthenticated()) return false; + return transport_->Send("report_metric_batch", codec_->EncodeMetricBatch(batch), + config_.timeout); +} + +std::shared_ptr CfmChannelPool::Next() const { + if (channels_.empty()) return nullptr; + const auto index = next_.fetch_add(1, std::memory_order_relaxed) % + channels_.size(); + return channels_[index]; +} + +bool CfmChannelPool::SendSnapshot(const IoPatternSnapshot& snapshot) { + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (channel && channel->SendSnapshot(snapshot)) return true; + } + return false; +} + +std::optional CfmChannelPool::PollPolicy() { + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (!channel) continue; + auto command = channel->PollPolicy(); + if (command) return command; + } + return std::nullopt; +} + +ErrorCode CfmChannelPool::ExecutePrefetch(const PrefetchPlan& plan) { + ErrorCode last_error = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (!channel) continue; + last_error = channel->ExecutePrefetch(plan); + if (last_error == ErrorCode::OK) return last_error; + } + return last_error; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp new file mode 100644 index 0000000000..f2a41f2cae --- /dev/null +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -0,0 +1,261 @@ +#include "io_pattern/runtime.h" + +#include +#include +#include + +namespace mooncake::io_pattern { + +IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) + : config_(config), + executor_(std::move(handlers.eviction), std::move(handlers.prefetch), + std::move(handlers.admission)), + feedback_(config.feedback_window) { + // A runtime always has an OOM guard even when a caller omits collector + // limits. The same bound is used by the bounded analyzer below. + if (config_.collector.max_total_keys == 0) { + config_.collector.max_total_keys = config_.max_analysis_keys; + } + if (config_.report_sink) { + reporter_ = std::make_shared( + config_.report_capacity, config_.report_sink, + config_.report_per_tenant_capacity); + reporter_->Start(); + } + collector_ = std::make_shared(config_.collector, + reporter_); + auto sliding = std::make_shared( + config.analysis_window_ns); + analyzer_ = std::make_shared(std::move(sliding)); + workload_policy_ = std::make_shared(); + std::shared_ptr legacy_strategy; + if (config_.legacy_fallback == LegacyFallback::kFifo) { + legacy_strategy = std::make_shared(); + } else { + legacy_strategy = std::make_shared(); + } + auto fallback = std::make_shared( + std::make_shared(std::move(legacy_strategy)), + nullptr, std::make_shared()); + policy_ = std::make_shared(workload_policy_, fallback); +} + +IoPatternRuntime::~IoPatternRuntime() { + if (reporter_) reporter_->Stop(); +} + +void IoPatternRuntime::ReportInferenceMetrics(const InferenceMetrics& metrics) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->ReportInferenceMetrics(metrics); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) + observability_.RecordReportDrop(dropped_after - dropped_before); +} + +void IoPatternRuntime::RecordAccess(const std::string& key, + const AccessRecord& record) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->RecordAccess(key, record); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) + observability_.RecordReportDrop(dropped_after - dropped_before); + + PolicyFeedbackSample feedback; + bool has_feedback = false; + { + std::lock_guard lock(feedback_state_mutex_); + ++feedback_accesses_; + feedback_hits_ += record.is_hit; + ObjectRef object = record.object; + if (!key.empty()) object.key = key; + if (pending_prefetches_.erase(object) != 0) { + feedback.prefetch_accuracy = record.is_hit ? 1.0F : 0.0F; + has_feedback = true; + if (!record.is_hit) observability_.RecordFalsePositive(); + } + // A completed 64-access window is a stable, bounded source of actual + // hit-rate deltas. TTFT remains supplied by the inference bridge via + // the public RecordFeedback API. + if (feedback_accesses_ >= 64) { + const auto hit_rate = static_cast(feedback_hits_) / + static_cast(feedback_accesses_); + feedback.hit_rate_delta = hit_rate - previous_hit_rate_; + previous_hit_rate_ = hit_rate; + feedback_accesses_ = 0; + feedback_hits_ = 0; + has_feedback = true; + } + } + if (has_feedback) RecordFeedback(feedback); +} + +void IoPatternRuntime::RecordStorageMetric(const StorageMetric& metric) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->RecordStorageMetric(metric); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) + observability_.RecordReportDrop(dropped_after - dropped_before); +} + +void IoPatternRuntime::MergeSnapshot(const IoPatternSnapshot& snapshot) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->MergeSnapshot(snapshot); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) { + observability_.RecordReportDrop(dropped_after - dropped_before); + } +} + +PatternResult IoPatternRuntime::AnalyzeWithinBudget( + const IoPatternSnapshot& snapshot, bool& degraded) { + degraded = config_.max_analysis_keys != 0 && + snapshot.keys.size() > config_.max_analysis_keys; + if (degraded || analysis_in_flight_->exchange(true, std::memory_order_acq_rel)) { + degraded = true; + return analyzer_->FallbackResult(); + } + + std::promise promise; + auto result = promise.get_future(); + auto analyzer = analyzer_; + auto in_flight = analysis_in_flight_; + std::thread([analyzer = std::move(analyzer), snapshot, + promise = std::move(promise), in_flight]() mutable { + try { + promise.set_value(analyzer->Analyze(snapshot)); + } catch (...) { + promise.set_value(analyzer->FallbackResult()); + } + in_flight->store(false, std::memory_order_release); + }).detach(); + + if (result.wait_for(std::chrono::microseconds(config_.analysis_timeout_us)) == + std::future_status::ready) { + return result.get(); + } + degraded = true; + return analyzer_->FallbackResult(); +} + +PolicyExecutionStatus IoPatternRuntime::Execute( + CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, + const std::vector& admissions, const std::string& session_id) { + const auto snapshot = collector_->GetSnapshot(); + const auto start = std::chrono::steady_clock::now(); + bool analysis_degraded = false; + const auto analysis = AnalyzeWithinBudget(snapshot, analysis_degraded); + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + observability_.RecordAnalyzeLatency(elapsed); + + workload_policy_->SetWorkloadType(analysis.workload_type); + workload_policy_->SetSessionWorkloads(analysis.sessions); + workload_policy_->AdvanceTransitionWindow(); + const PolicyResult result = policy_->ExecutePolicy( + PolicyContext{.snapshot = snapshot, .analysis = analysis, + .session_id = session_id}, eviction_tier, + eviction_bytes, trace, admissions); + auto status = executor_.Execute(result); + status.degraded = status.degraded || result.degraded || collector_->degraded() || + analysis_degraded || + elapsed > static_cast(config_.analysis_timeout_us); + observability_.RecordPolicyDecision(!result.eviction.candidates.empty() || + !result.prefetch.candidates.empty()); + const bool failed = status.eviction != ErrorCode::OK || + status.prefetch != ErrorCode::OK || status.degraded; + if (failed) policy_->RecordFailure(); + else policy_->RecordSuccess(); + if (status.degraded || policy_->degraded()) observability_.RecordDegrade(); + status.degraded = status.degraded || policy_->degraded(); + + PolicyFeedbackSample feedback; + bool has_feedback = false; + { + std::lock_guard lock(feedback_state_mutex_); + for (const auto& candidate : result.prefetch.candidates) { + if (config_.max_pending_prefetches == 0 || + pending_prefetches_.size() < config_.max_pending_prefetches) { + pending_prefetches_.insert(candidate.object); + } + } + if (!result.prefetch.candidates.empty() && status.prefetch != ErrorCode::OK) { + feedback.prefetch_accuracy = 0.0F; + has_feedback = true; + } + if (!snapshot.keys.empty() && !result.eviction.candidates.empty()) { + feedback.eviction_churn = static_cast( + result.eviction.candidates.size()) / + static_cast(snapshot.keys.size()); + has_feedback = true; + } + } + if (has_feedback) RecordFeedback(feedback); + return status; +} + +ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { + PolicyResult result; + if (const auto* eviction = std::get_if(&command)) { + result.eviction = *eviction; + } else if (const auto* prefetch = std::get_if(&command)) { + result.prefetch = *prefetch; + } else { + result.admissions.push_back(std::get(command)); + } + const auto status = executor_.Execute(result); + if (status.degraded) { + observability_.RecordDegrade(); + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + if (const auto* eviction = std::get_if(&command)) { + return status.eviction; + } + if (const auto* prefetch = std::get_if(&command)) { + return status.prefetch; + } + return status.admissions.empty() ? ErrorCode::OK : status.admissions.front(); +} + +void IoPatternRuntime::RecordFeedback(PolicyFeedbackSample sample) { + feedback_.Record(sample); + auto config = workload_policy_->CurrentEvictionConfig(); + if (tuner_.Tune(feedback_.Snapshot(), config)) { + workload_policy_->ApplyEvictionTuning(config); + } +} + +IoPatternSnapshot IoPatternRuntime::Snapshot() const { + return collector_->GetSnapshot(); +} + +IoPatternObservabilitySnapshot IoPatternRuntime::ObservabilitySnapshot( + double window_seconds) const { + return observability_.Snapshot(window_seconds); +} + +bool IoPatternRuntime::degraded() const { + return collector_->degraded() || analyzer_->degraded() || policy_->degraded(); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp new file mode 100644 index 0000000000..ed992fc45e --- /dev/null +++ b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp @@ -0,0 +1,92 @@ +#include "io_pattern/sliding_window_analyzer.h" + +#include +#include + +namespace mooncake::io_pattern { +namespace { +template +T Percentile(std::vector values, size_t rank) { + if (values.empty()) return 0; + std::sort(values.begin(), values.end()); + return values[std::min(rank, values.size() - 1)]; +} +} + +void SlidingWindowAnalyzer::Append(const IoPatternSnapshot& snapshot) const { + std::lock_guard lock(mutex_); + if (history_.empty() || + history_.back().generated_at_ns != snapshot.generated_at_ns) { + history_.push_back(snapshot); + } + const uint64_t cutoff = snapshot.generated_at_ns > window_ns_ + ? snapshot.generated_at_ns - window_ns_ + : 0; + while (!history_.empty() && history_.front().generated_at_ns < cutoff) + history_.pop_front(); +} + +IoPatternSnapshot SlidingWindowAnalyzer::Aggregate( + const IoPatternSnapshot& current) const { + Append(current); + std::lock_guard lock(mutex_); + IoPatternSnapshot aggregate = current; + aggregate.keys.clear(); + for (const auto& snapshot : history_) { + aggregate.keys.insert(aggregate.keys.end(), snapshot.keys.begin(), + snapshot.keys.end()); + } + return aggregate; +} + +PatternResult SlidingWindowAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + const auto aggregate = Aggregate(snapshot); + auto result = analyzer_.Analyze(aggregate); + return result.workload_type == WorkloadType::kMixed + ? kmeans_.Analyze(aggregate) + : result; +} + +WorkloadType SlidingWindowAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + return Analyze(snapshot).workload_type; +} + +float SlidingWindowAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto result = Analyze(snapshot); + const auto it = std::find_if(result.keys.begin(), result.keys.end(), + [&object](const KeyPattern& key) { + return key.object == object; + }); + return it == result.keys.end() ? 0.0F : it->confidence; +} + +WorkloadFeatureStats SlidingWindowAnalyzer::FeatureStats() const { + std::lock_guard lock(mutex_); + std::vector tokens, fanouts, matches, frequencies; + std::vector blocks; + for (const auto& snapshot : history_) { + for (const auto& key : snapshot.keys) { + tokens.push_back(key.token_count); + fanouts.push_back(key.prefix_fanout); + matches.push_back(key.match_length); + frequencies.push_back(static_cast(key.access_count_window)); + blocks.push_back(key.block_size); + } + } + const auto p90 = [](size_t size) { return size == 0 ? 0 : (size * 9) / 10; }; + WorkloadFeatureStats stats; + stats.samples = tokens.size(); + stats.token_median = Percentile(tokens, tokens.size() / 2); + stats.token_p90 = Percentile(tokens, p90(tokens.size())); + stats.fanout_p90 = Percentile(fanouts, p90(fanouts.size())); + stats.block_median = Percentile(blocks, blocks.size() / 2); + stats.block_p90 = Percentile(blocks, p90(blocks.size())); + stats.match_p90 = Percentile(matches, p90(matches.size())); + stats.frequency_median = Percentile(frequencies, frequencies.size() / 2); + return stats; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/threshold_analyzer.cpp b/mooncake-store/src/io_pattern/threshold_analyzer.cpp new file mode 100644 index 0000000000..f84d8a2c1b --- /dev/null +++ b/mooncake-store/src/io_pattern/threshold_analyzer.cpp @@ -0,0 +1,134 @@ +#include "io_pattern/threshold_analyzer.h" + +#include + +namespace mooncake::io_pattern { +namespace { + +bool IsCodeAgent(const KeyMetrics& key, const ThresholdAnalyzerConfig& config) { + return key.token_count > config.code_agent_token_count && + key.prefix_fanout > config.code_agent_prefix_fanout && + key.match_length > config.code_agent_match_length; +} + +bool IsRecommendation(const KeyMetrics& key, + const ThresholdAnalyzerConfig& config) { + return key.block_size < config.recommendation_block_size && + key.access_count_window > config.recommendation_frequency; +} + +bool IsConversation(const KeyMetrics& key, + const ThresholdAnalyzerConfig& config) { + return key.prefix_fanout > config.conversation_prefix_fanout && + key.match_length > config.conversation_match_length; +} + +float RuleConfidence(const KeyMetrics& key, + const ThresholdAnalyzerConfig& config) { + float score = 0.0F; + if (IsCodeAgent(key, config)) score = std::max(score, 1.0F); + if (IsRecommendation(key, config)) score = std::max(score, 1.0F); + if (IsConversation(key, config)) score = std::max(score, 1.0F); + // A partial match is useful to policies, but must not look like a + // definitive workload classification. + if (score == 0.0F) { + const float code = std::min( + {static_cast(key.token_count) / + std::max(1.0F, static_cast(config.code_agent_token_count)), + static_cast(key.prefix_fanout) / + std::max(1.0F, static_cast(config.code_agent_prefix_fanout)), + static_cast(key.match_length) / + std::max(1.0F, static_cast(config.code_agent_match_length))}); + const float recommendation = std::min( + static_cast(config.recommendation_block_size) / + std::max(1.0F, static_cast(key.block_size)), + static_cast(key.access_count_window) / + std::max(1.0F, static_cast(config.recommendation_frequency))); + const float conversation = std::min( + static_cast(key.prefix_fanout) / + std::max(1.0F, static_cast(config.conversation_prefix_fanout)), + static_cast(key.match_length) / + std::max(1.0F, static_cast(config.conversation_match_length))); + score = std::clamp(std::max({code, recommendation, conversation}), + 0.0F, 1.0F); + } + return score; +} + +} // namespace + +PatternResult ThresholdAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + PatternResult result; + result.workload_type = DetectWorkloadType(snapshot); + if (!snapshot.keys.empty()) { + float total = 0.0F; + for (const auto& key : snapshot.keys) total += KeyConfidence(key); + result.workload_confidence = + std::clamp(total / static_cast(snapshot.keys.size()), 0.0F, + 1.0F); + } + result.keys.reserve(snapshot.keys.size()); + for (const auto& key : snapshot.keys) { + KeyPattern pattern; + pattern.object = key.object; + pattern.confidence = KeyConfidence(key); + pattern.frequency_score = std::min( + 1.0F, static_cast(key.access_count_window) / 20.0F); + pattern.idle_score = std::min( + 1.0F, static_cast(key.idle_time_us) / 1'000'000.0F); + pattern.prefix_score = std::min( + 1.0F, static_cast(key.match_length) / 256.0F); + pattern.recompute_score = std::min(1.0F, key.recompute_cost); + pattern.transfer_roi = key.transfer_eta_us == 0 + ? 0.0F + : key.recompute_cost / + static_cast(key.transfer_eta_us); + pattern.migration_safe = !key.pinned; + result.keys.push_back(pattern); + } + return result; +} + +WorkloadType ThresholdAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + if (snapshot.keys.empty()) { + return WorkloadType::kMixed; + } + uint32_t code_agents = 0; + uint32_t recommendations = 0; + uint32_t conversations = 0; + for (const auto& key : snapshot.keys) { + code_agents += IsCodeAgent(key, config_); + recommendations += IsRecommendation(key, config_); + conversations += IsConversation(key, config_); + } + const uint32_t matched = static_cast(code_agents != 0) + + static_cast(recommendations != 0) + + static_cast(conversations != 0); + if (matched > 1) return WorkloadType::kMixed; + if (code_agents != 0) return WorkloadType::kCodeAgent; + if (recommendations != 0) return WorkloadType::kGenerativeRecommendation; + if (conversations != 0) return WorkloadType::kMultiTurnConversation; + return WorkloadType::kMixed; +} + +float ThresholdAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto* key = FindKey(object, snapshot); + return key == nullptr ? 0.0F : KeyConfidence(*key); +} + +const KeyMetrics* ThresholdAnalyzer::FindKey( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto it = std::find_if( + snapshot.keys.begin(), snapshot.keys.end(), + [&object](const KeyMetrics& key) { return key.object == object; }); + return it == snapshot.keys.end() ? nullptr : &*it; +} + +float ThresholdAnalyzer::KeyConfidence(const KeyMetrics& key) const { + return RuleConfidence(key, config_); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/tier_executor.cpp b/mooncake-store/src/io_pattern/tier_executor.cpp new file mode 100644 index 0000000000..02ba4371de --- /dev/null +++ b/mooncake-store/src/io_pattern/tier_executor.cpp @@ -0,0 +1,35 @@ +#include "io_pattern/tier_executor.h" + +namespace mooncake::io_pattern { + +PolicyExecutionStatus TierOperationExecutor::Execute( + const PolicyResult& result) const { + PolicyExecutionStatus status; + if (eviction_ && (!result.eviction.candidates.empty() || + result.eviction.target_bytes != 0)) { + status.eviction = eviction_(result.eviction); + } else if (!result.eviction.candidates.empty() || + result.eviction.target_bytes != 0) { + status.eviction = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + status.degraded = true; + } + if (prefetch_ && !result.prefetch.candidates.empty()) { + status.prefetch = prefetch_(result.prefetch); + } else if (!result.prefetch.candidates.empty()) { + status.prefetch = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + status.degraded = true; + } + for (const auto& admission : result.admissions) { + if (admission_ && admission.decision == AdmissionDecision::kAdmit) { + status.admissions.push_back(admission_(admission)); + } else if (admission.decision == AdmissionDecision::kAdmit) { + status.admissions.push_back(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + status.degraded = true; + } else { + status.admissions.push_back(ErrorCode::OK); + } + } + return status; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index d5660e8e74..49f8be3c10 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -57,6 +57,7 @@ #include "master_snapshot_repository.h" #include "ha_metric_manager.h" #include "metadata_store.h" +#include "io_pattern/runtime.h" namespace mooncake { @@ -412,6 +413,51 @@ MasterService::MasterService(const MasterServiceConfig& config) << ")"; } + io_pattern_runtime_ = std::make_unique( + io_pattern::IoPatternRuntime::Handlers{ + .eviction = [this](const io_pattern::EvictionPlan& plan) { + bool evicted = plan.candidates.empty(); + std::unordered_map targets; + for (const auto& candidate : plan.candidates) { + targets[candidate.object.tenant_id] += candidate.bytes; + } + for (const auto& [tenant, bytes] : targets) { + const auto result = EvictTenantMemoryForQuota(tenant, bytes); + evicted = evicted || result.freed_bytes != 0; + } + return evicted ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; + }, + .prefetch = [this](const io_pattern::PrefetchPlan& plan) { + for (const auto& candidate : plan.candidates) { + // Store's safe promotion primitive is LOCAL_DISK -> MEMORY; + // HBM remains inference-runtime-owned and is never promoted + // from the master control plane. + if (candidate.target_tier == io_pattern::CacheTier::kL0Hbm) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + const ObjectIdentity object_id{candidate.object.tenant_id, + candidate.object.key}; + if (TryPushPromotionQueue(object_id, + /*record_candidate=*/false) != + PromotionQueueResult::kQueued) { + return ErrorCode::OBJECT_NOT_FOUND; + } + } + return ErrorCode::OK; + }, + .admission = [this](const io_pattern::AdmissionResult& result) { + if (result.target_tier == io_pattern::CacheTier::kL0Hbm) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + const ObjectIdentity object_id{result.object.tenant_id, + result.object.key}; + return TryPushPromotionQueue(object_id, + /*record_candidate=*/false) == + PromotionQueueResult::kQueued + ? ErrorCode::OK + : ErrorCode::OBJECT_NOT_FOUND; + }}); + kv_event_publisher_ = std::make_unique(BuildKvEventConfig(config)); @@ -3185,6 +3231,8 @@ auto MasterService::GetReplicaList(const std::string& key, GetReplicaListResponse resp({}, default_kv_lease_ttl_); bool promotion_eligible = false; + io_pattern::AccessRecord io_access; + bool record_io_access = false; { MetadataAccessorRO accessor(this, object_id); @@ -3262,11 +3310,26 @@ auto MasterService::GetReplicaList(const std::string& key, resp = GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_, metadata.object_checksum); + io_access.object = {object_id.tenant_id, key}; + io_access.observed_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + io_access.block_size = metadata.size; + io_access.tier = resp.replicas[0].is_memory_replica() + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd; + io_access.operation = io_pattern::IoOperation::kGet; + io_access.is_hit = true; + record_io_access = true; } // RO accessor released. Safe to take a fresh RW accessor now. if (promotion_eligible) { TryPushPromotionQueue(object_id); } + if (record_io_access && io_pattern_runtime_) { + io_pattern_runtime_->RecordAccess(key, io_access); + } return resp; } @@ -3344,6 +3407,7 @@ MasterService::BatchGetReplicaList(const std::vector& keys, } std::vector promotion_candidates; + std::vector io_accesses; std::shared_lock shared_lock(snapshot_mutex_); { MetadataShardAccessorRO shard(this, shard_idx); @@ -3423,12 +3487,29 @@ MasterService::BatchGetReplicaList(const std::vector& keys, results[original_idx] = GetReplicaListResponse( std::move(replica_list), default_kv_lease_ttl_, metadata.object_checksum); + io_accesses.push_back( + {.object = {normalized_tenant, key}, + .observed_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()), + .block_size = metadata.size, + .tier = results[original_idx]->replicas[0].is_memory_replica() + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd, + .operation = io_pattern::IoOperation::kGet, + .is_hit = true}); } } for (const auto& object_id : promotion_candidates) { TryPushPromotionQueue(object_id); } + if (io_pattern_runtime_) { + for (const auto& access : io_accesses) { + io_pattern_runtime_->RecordAccess(access.object.key, access); + } + } } return results; @@ -4076,6 +4157,22 @@ auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, metadata.GrantLease(0, default_kv_soft_pin_ttl_); PublishKvStored(key, replica_type, metadata, object_id.tenant_id); + if (io_pattern_runtime_) { + io_pattern_runtime_->RecordAccess( + key, {.object = {object_id.tenant_id, key}, + .observed_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()), + .block_size = metadata.size, + .tier = replica_type == ReplicaType::MEMORY + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd, + .operation = io_pattern::IoOperation::kPut, + .is_hit = true, + .write_batch_size = 1}); + } + if (enable_oplog_ && ordered_oplog_writer_) { std::string payload = SerializeMetadataForOpLog(metadata); auto result = AppendOpLogVisibleBeforeDurable( @@ -7433,6 +7530,16 @@ void MasterService::EvictionThreadFunc() { double evict_ratio_lowerbound = std::max(evict_ratio_target * 0.5, used_ratio - eviction_high_watermark_ratio_); + if (io_pattern_runtime_) { + io_pattern_runtime_->RecordStorageMetric( + {.source_id = "master-memory", .tier = io_pattern::CacheTier::kL1Host, + .memory_used_ratio = static_cast(used_ratio)}); + const auto capacity = std::max( + 0, MasterMetricManager::instance().get_total_mem_capacity()); + io_pattern_runtime_->Execute( + io_pattern::CacheTier::kL1Host, + static_cast(evict_ratio_target * capacity), {}); + } BatchEvict(evict_ratio_target, evict_ratio_lowerbound); LOG(INFO) << "[EVICT-DONE] BatchEvict execution completed."; last_discard_time = now; @@ -8035,7 +8142,7 @@ MasterService::TenantQuotaEvictionResult MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, uint64_t target_bytes) { TenantQuotaEvictionResult total; - if (!enable_multi_tenants_ || target_bytes == 0) { + if (target_bytes == 0) { return total; } diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index e35e2c6dd7..b5836d306f 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1,9 +1,12 @@ #include "io_pattern/io_pattern.h" +#include "io_pattern/threshold_analyzer.h" +#include "io_pattern/policy_strategies.h" #include #include #include #include +#include #include @@ -25,7 +28,7 @@ class TestCollector final : public IoPatternCollector { void ReportInferenceMetrics(const InferenceMetrics& metrics) override { inference_metrics = metrics; } - void RecordAccess(const AccessRecord& record) override { + void RecordAccess(const std::string&, const AccessRecord& record) override { access_record = record; } void RecordStorageMetric(const StorageMetric& metric) override { @@ -57,6 +60,19 @@ class TestAnalyzer final : public IoPatternAnalyzer { .workload_confidence = 0.75F}; }; +class ThrowingAnalyzer final : public IoPatternAnalyzer { + public: + PatternResult Analyze(const IoPatternSnapshot&) const override { + throw std::runtime_error("analysis failure"); + } + WorkloadType DetectWorkloadType(const IoPatternSnapshot&) const override { + throw std::runtime_error("analysis failure"); + } + float CalculateConfidence(const ObjectRef&, const IoPatternSnapshot&) const override { + throw std::runtime_error("analysis failure"); + } +}; + class TestPrefetchOps final : public PrefetchOps { public: PrefetchPlan Evaluate(const PolicyContext&, @@ -87,11 +103,75 @@ class TestPrefetchExecutor final : public PrefetchExecutor { PrefetchPlan plan; }; +class TestCfmChannel final : public CfmChannel { + public: + bool SendSnapshot(const IoPatternSnapshot& value) override { + snapshot = value; + return send_ok; + } + std::optional PollPolicy() override { return policy; } + ErrorCode ExecutePrefetch(const PrefetchPlan& value) override { + plan = value; + return execute_code; + } + bool send_ok{true}; + ErrorCode execute_code{ErrorCode::OK}; + IoPatternSnapshot snapshot; + std::optional policy; + PrefetchPlan plan; +}; + +class FlakyCfmChannel final : public CfmChannel { + public: + bool SendSnapshot(const IoPatternSnapshot&) override { + return send_failures-- <= 0; + } + std::optional PollPolicy() override { return PrefetchPlan{}; } + ErrorCode ExecutePrefetch(const PrefetchPlan&) override { + return ErrorCode::RPC_FAIL; + } + int send_failures{0}; +}; + +class TestRpcTransport final : public CfmRpcTransport { + public: + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) override { + last_method = std::string(method); + last_payload = std::string(payload); + last_timeout = timeout; + return send_ok; + } + std::optional Receive(std::string_view method, + std::chrono::milliseconds timeout) override { + last_method = std::string(method); + last_timeout = timeout; + return response; + } + bool send_ok{true}; + std::optional response; + std::string last_method; + std::string last_payload; + std::chrono::milliseconds last_timeout{0}; +}; + +class TestRpcCodec final : public CfmRpcCodec { + public: + std::string EncodeSnapshot(const IoPatternSnapshot&) const override { return "snapshot"; } + std::string EncodePrefetch(const PrefetchPlan&) const override { return "prefetch"; } + std::string EncodeMetricBatch(const MetricBatch&) const override { + return "batch"; + } + std::optional DecodePolicy(const std::string& value) const override { + return value == "policy" ? std::optional(PrefetchPlan{}) + : std::nullopt; + } +}; + TEST(IoPatternFrameworkTest, PublicSeamsRemainAbstract) { static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); - static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); @@ -144,7 +224,7 @@ TEST(IoPatternFrameworkTest, CollectorAndAnalyzerExposeValueFlow) { access_record.object = collector.inference_metrics.object; access_record.observed_at_ns = 43; access_record.is_hit = true; - collector.RecordAccess(access_record); + collector.RecordAccess("key", access_record); collector.RecordStorageMetric(StorageMetric{.source_id = "segment-1"}); const auto snapshot = collector.GetSnapshot(); @@ -160,6 +240,268 @@ TEST(IoPatternFrameworkTest, CollectorAndAnalyzerExposeValueFlow) { EXPECT_EQ(analyzer.DetectWorkloadType(snapshot), WorkloadType::kMixed); } +TEST(IoPatternFrameworkTest, CollectorImplAggregatesAndIsolatesTenants) { + IoPatternCollectorImpl collector; + AccessRecord access; + access.object = {TenantId("tenant-a"), "ignored"}; + access.observed_at_ns = 200; + access.block_size = 4096; + access.tier = CacheTier::kL1Host; + access.is_hit = true; + collector.RecordAccess("shared-key", access); + access.object.tenant_id = TenantId("tenant-b"); + access.observed_at_ns = 100; + collector.RecordAccess("shared-key", access); + + const auto snapshot = collector.GetSnapshot(); + ASSERT_EQ(snapshot.keys.size(), 2); + EXPECT_EQ(snapshot.keys[0].object.tenant_id.value(), "tenant-a"); + EXPECT_EQ(snapshot.keys[1].object.tenant_id.value(), "tenant-b"); + EXPECT_EQ(snapshot.keys[0].object.key, "shared-key"); +} + +TEST(IoPatternFrameworkTest, CollectorImplKeepsLatestStorageObservation) { + IoPatternCollectorImpl collector; + collector.RecordStorageMetric(StorageMetric{.source_id = "segment", + .observed_at_ns = 20, + .used_bytes = 200}); + collector.RecordStorageMetric(StorageMetric{.source_id = "segment", + .observed_at_ns = 10, + .used_bytes = 100}); + const auto snapshot = collector.GetSnapshot(); + ASSERT_EQ(snapshot.storage.size(), 1); + EXPECT_EQ(snapshot.storage.front().used_bytes, 200); +} + +TEST(IoPatternFrameworkTest, CollectorImplEnforcesPerTenantKeyQuota) { + IoPatternCollectorImpl collector( + IoPatternCollectorImpl::Config{.max_keys_per_tenant = 1}); + InferenceMetrics first; + first.object = {TenantId("tenant-a"), "first"}; + collector.ReportInferenceMetrics(first); + InferenceMetrics second; + second.object = {TenantId("tenant-a"), "second"}; + collector.ReportInferenceMetrics(second); + InferenceMetrics other_tenant; + other_tenant.object = {TenantId("tenant-b"), "second"}; + collector.ReportInferenceMetrics(other_tenant); + EXPECT_EQ(collector.GetSnapshot().keys.size(), 2); + EXPECT_EQ(collector.dropped(), 1); +} + +TEST(IoPatternFrameworkTest, CollectorImplDegradesAtGlobalKeyLimit) { + IoPatternCollectorImpl collector( + IoPatternCollectorImpl::Config{.max_total_keys = 1}); + InferenceMetrics first; + first.object = {TenantId("tenant-a"), "first"}; + collector.ReportInferenceMetrics(first); + InferenceMetrics second; + second.object = {TenantId("tenant-b"), "second"}; + collector.ReportInferenceMetrics(second); + EXPECT_TRUE(collector.degraded()); + EXPECT_EQ(collector.dropped(), 1); + EXPECT_EQ(collector.GetSnapshot().keys.size(), 1); + collector.RecordStorageMetric(StorageMetric{.source_id = "segment"}); + EXPECT_EQ(collector.GetSnapshot().storage.size(), 1); +} + +TEST(IoPatternFrameworkTest, CollectorImplFeedsReporterWithoutInlineTransport) { + MetricBatch received; + auto reporter = std::make_shared(4, [&](const MetricBatch& batch) { + received = batch; + return true; + }); + IoPatternCollectorImpl collector({}, reporter); + collector.ReportInferenceMetrics(InferenceMetrics{}); + collector.RecordStorageMetric(StorageMetric{}); + EXPECT_EQ(reporter->pending(), 2); + EXPECT_TRUE(collector.FlushReports()); + EXPECT_EQ(received.inference.size(), 1); + EXPECT_EQ(received.storage.size(), 1); +} + +TEST(IoPatternFrameworkTest, CollectorImplDerivesWritePathMetrics) { + IoPatternCollectorImpl collector; + AccessRecord write; + write.object = {TenantId("tenant-a"), "write-key"}; + write.operation = IoOperation::kPut; + write.block_size = 4096; + write.write_batch_size = 32; + collector.RecordAccess("write-key", write); + write.overwrite = true; + collector.RecordAccess("write-key", write); + const auto snapshot = collector.GetSnapshot(); + const auto& key = snapshot.keys.front(); + EXPECT_EQ(key.write_frequency, 2); + EXPECT_EQ(key.write_batch_size, 32); + EXPECT_EQ(key.write_object_size, 4096); + EXPECT_FLOAT_EQ(key.overwrite_ratio, 0.5F); + EXPECT_TRUE(key.write_burst); +} + +TEST(IoPatternFrameworkTest, ThresholdAnalyzerClassifiesDocumentedWorkloads) { + ThresholdAnalyzer analyzer; + IoPatternSnapshot code_agent; + KeyMetrics code_key; + code_key.object = {TenantId("tenant-a"), "code"}; + code_key.token_count = 20 * 1024; + code_key.prefix_fanout = 20; + code_key.match_length = 512; + code_agent.keys.push_back(code_key); + EXPECT_EQ(analyzer.DetectWorkloadType(code_agent), + WorkloadType::kCodeAgent); + + IoPatternSnapshot recommendation; + KeyMetrics recommendation_key; + recommendation_key.object = {TenantId("tenant-a"), "recommendation"}; + recommendation_key.block_size = 64 * 1024; + recommendation_key.access_count_window = 30; + recommendation.keys.push_back(recommendation_key); + EXPECT_EQ(analyzer.DetectWorkloadType(recommendation), + WorkloadType::kGenerativeRecommendation); +} + +TEST(IoPatternFrameworkTest, ThresholdAnalyzerFallsBackToMixed) { + ThresholdAnalyzer analyzer; + IoPatternSnapshot snapshot; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "unknown"}; + snapshot.keys.push_back(key); + + const auto result = analyzer.Analyze(snapshot); + EXPECT_EQ(result.workload_type, WorkloadType::kMixed); + EXPECT_FLOAT_EQ(result.workload_confidence, 0.0F); + ASSERT_EQ(result.keys.size(), 1); + EXPECT_EQ(result.keys.front().object.key, "unknown"); + EXPECT_FLOAT_EQ(analyzer.CalculateConfidence(key.object, snapshot), 0.0F); +} + +TEST(IoPatternFrameworkTest, ThresholdAnalyzerReportsMixedAndPartialConfidence) { + ThresholdAnalyzer analyzer; + IoPatternSnapshot snapshot; + KeyMetrics code; + code.object = {TenantId("tenant-a"), "code"}; + code.token_count = 20 * 1024; + code.prefix_fanout = 20; + code.match_length = 512; + snapshot.keys.push_back(code); + KeyMetrics recommendation; + recommendation.object = {TenantId("tenant-b"), "recommendation"}; + recommendation.block_size = 64 * 1024; + recommendation.access_count_window = 30; + snapshot.keys.push_back(recommendation); + EXPECT_EQ(analyzer.DetectWorkloadType(snapshot), WorkloadType::kMixed); + EXPECT_FLOAT_EQ(analyzer.CalculateConfidence( + {TenantId("tenant-a"), "code"}, snapshot), + 1.0F); + KeyMetrics partial; + partial.object = {TenantId("tenant-c"), "partial"}; + partial.token_count = 8 * 1024; + snapshot.keys.push_back(partial); + EXPECT_GT(analyzer.CalculateConfidence(partial.object, snapshot), 0.0F); +} + +TEST(IoPatternFrameworkTest, ScoreEvictionSelectsColdObjectsWithinBudget) { + ScoreBasedEvictionOps eviction; + PolicyContext context; + KeyMetrics cold; + cold.object = {TenantId("tenant-a"), "cold"}; + cold.block_size = 100; + cold.replica_tiers = CacheTierBit(CacheTier::kL1Host); + KeyMetrics hot = cold; + hot.object.key = "hot"; + hot.block_size = 100; + context.snapshot.keys = {cold, hot}; + context.analysis.keys = { + {.object = cold.object, .frequency_score = 0.1F, .idle_score = 0.9F}, + {.object = hot.object, .frequency_score = 0.9F, .idle_score = 0.1F}, + }; + + const auto plan = eviction.Evaluate(context, CacheTier::kL1Host, 100); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().object.key, "cold"); + EXPECT_EQ(plan.candidates.front().bytes, 100); +} + +TEST(IoPatternFrameworkTest, ScoreEvictionSkipsPinnedAndZeroBudget) { + ScoreBasedEvictionOps eviction; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "pinned"}; + key.block_size = 1; + key.pinned = true; + key.replica_tiers = CacheTierBit(CacheTier::kL1Host); + context.snapshot.keys.push_back(key); + context.analysis.keys.push_back( + KeyPattern{.object = key.object, .frequency_score = 0.0F}); + + EXPECT_TRUE(eviction.Evaluate(context, CacheTier::kL1Host, 1024) + .candidates.empty()); + key.pinned = false; + context.snapshot.keys.front() = key; + EXPECT_TRUE(eviction.Evaluate(context, CacheTier::kL1Host, 0) + .candidates.empty()); +} + +TEST(IoPatternFrameworkTest, PrefixAdmissionUsesTierSpecificSignals) { + PrefixMatchAdmissionOps admission; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "prefix"}; + key.access_count_window = 10; + key.match_length = 64; + context.snapshot.keys.push_back(key); + + const auto hbm = admission.Evaluate(key.object, CacheTier::kL0Hbm, context); + EXPECT_EQ(hbm.decision, AdmissionDecision::kAdmit); + + key.match_length = 1; + context.snapshot.keys.front() = key; + const auto rejected = + admission.Evaluate(key.object, CacheTier::kL0Hbm, context); + EXPECT_EQ(rejected.decision, AdmissionDecision::kRejectPrefix); +} + +TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { + TraceBasedPrefetchOps prefetch; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "block"}; + key.block_size = 4096; + key.replica_tiers = CacheTierBit(CacheTier::kL3NofSsd); + context.snapshot.keys.push_back(key); + + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 512, .is_hit = true}); + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 8, .is_hit = true}); + + const auto plan = prefetch.Evaluate(context, trace); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().source_tier, CacheTier::kL3NofSsd); + EXPECT_EQ(plan.candidates.front().target_tier, CacheTier::kL2Segment); + EXPECT_EQ(plan.candidates.front().bytes, 4096); +} + +TEST(IoPatternFrameworkTest, TracePrefetchDeduplicatesObjects) { + TraceBasedPrefetchOps prefetch; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "block"}; + key.block_size = 128; + key.replica_tiers = CacheTierBit(CacheTier::kL2Segment); + context.snapshot.keys.push_back(key); + + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 300, .is_hit = true}); + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 400, .is_hit = true}); + + EXPECT_EQ(prefetch.Evaluate(context, trace).candidates.size(), 1); +} + TEST(IoPatternFrameworkTest, PolicyContextCarriesRawAndDerivedViews) { PolicyContext context; context.snapshot.generated_at_ns = 123; @@ -242,5 +584,614 @@ TEST(IoPatternFrameworkTest, ComposedEngineDelegatesAndDegradesSafely) { EXPECT_EQ(executor.plan.strategy, PrefetchStrategy::kWaitComplete); } +TEST(IoPatternFrameworkTest, WorkloadPolicyEngineSelectsAndTransitionsTemplates) { + WorkloadPolicyEngine engine(WorkloadType::kCodeAgent, 3); + EXPECT_EQ(engine.ActiveWorkload(), WorkloadType::kCodeAgent); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 1.0F); + + engine.SetWorkloadType(WorkloadType::kGenerativeRecommendation); + EXPECT_EQ(engine.ActiveWorkload(), + WorkloadType::kGenerativeRecommendation); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 0.0F); + engine.AdvanceTransitionWindow(); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 1.0F / 3.0F); + engine.AdvanceTransitionWindow(); + engine.AdvanceTransitionWindow(); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 1.0F); + + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "item"}; + key.block_size = 64 * 1024; + key.access_count_window = 30; + context.snapshot.keys.push_back(key); + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 64, .is_hit = true}); + EXPECT_EQ(engine.PlanPrefetch(context, trace).strategy, + PrefetchStrategy::kWaitComplete); +} + +TEST(IoPatternFrameworkTest, UnifiedPolicyResultSeamDelegates) { + IoPatternSnapshot snapshot; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "key"}; + snapshot.keys.push_back(key); + ComposedPolicyEngine engine(std::make_shared(), + std::make_shared(), + std::make_shared()); + PolicyContext context; + context.snapshot = snapshot; + const auto result = engine.ExecutePolicy( + context, CacheTier::kL1Host, 1024, {}, {key.object}); + EXPECT_EQ(result.admissions.size(), 1); + EXPECT_EQ(result.admissions.front().object, key.object); +} + +TEST(IoPatternFrameworkTest, RegistryPolicyEngineResolvesNamedOps) { + auto registries = std::make_shared(); + ASSERT_TRUE(registries->eviction.Register( + "score", [] { return std::make_shared(); })); + ASSERT_TRUE(registries->prefetch.Register( + "trace", [] { return std::make_shared(); })); + ASSERT_TRUE(registries->admission.Register( + "prefix", [] { return std::make_shared(); })); + RegistryPolicyEngine engine(registries, "score", "trace", "prefix"); + const ObjectRef object{TenantId("tenant-a"), "key"}; + const auto result = engine.ExecutePolicy({}, CacheTier::kL1Host, 1024, {}, + {object}); + ASSERT_EQ(result.admissions.size(), 1); + EXPECT_EQ(result.admissions.front().object, object); + + RegistryPolicyEngine missing(registries, "missing", "trace", "prefix"); + EXPECT_TRUE(missing.ExecutePolicy({}, CacheTier::kL1Host, 0, {}).degraded); +} + +TEST(IoPatternFrameworkTest, ReporterBatchesBoundsAndCountsDrops) { + MetricBatch received; + IoPatternReporter reporter(2, [&](const MetricBatch& batch) { + received = batch; + return true; + }); + EXPECT_TRUE(reporter.Enqueue(InferenceMetrics{})); + EXPECT_TRUE(reporter.EnqueueStorage(StorageMetric{})); + EXPECT_FALSE(reporter.EnqueueAccess(AccessRecord{})); + EXPECT_EQ(reporter.dropped(), 1); + EXPECT_EQ(reporter.pending(), 2); + EXPECT_TRUE(reporter.Flush()); + EXPECT_EQ(reporter.pending(), 0); + EXPECT_EQ(reporter.reported(), 2); + EXPECT_EQ(received.inference.size(), 1); + EXPECT_EQ(received.storage.size(), 1); +} + +TEST(IoPatternFrameworkTest, ReporterAdaptsFlushIntervalToLoad) { + IoPatternReporter reporter(4, [](const MetricBatch&) { return true; }); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(1000)); + reporter.Enqueue(InferenceMetrics{}); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(500)); + reporter.Enqueue(InferenceMetrics{}); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(100)); +} + +TEST(IoPatternFrameworkTest, ReporterEnforcesPerTenantFairness) { + IoPatternReporter reporter(4, [](const MetricBatch&) { return true; }, 1); + InferenceMetrics first; + first.object = {TenantId("tenant-a"), "a"}; + InferenceMetrics second = first; + second.object.key = "b"; + InferenceMetrics other = first; + other.object.tenant_id = TenantId("tenant-b"); + EXPECT_TRUE(reporter.Enqueue(first)); + EXPECT_FALSE(reporter.Enqueue(second)); + EXPECT_TRUE(reporter.Enqueue(other)); + EXPECT_EQ(reporter.dropped(), 1); +} + +TEST(IoPatternFrameworkTest, CfmClientDelegatesToTransportChannel) { + auto channel = std::make_shared(); + CfmClientImpl client(channel); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = 42; + EXPECT_EQ(client.ReportSnapshot(snapshot), ErrorCode::OK); + EXPECT_EQ(channel->snapshot.generated_at_ns, 42); + channel->policy = PrefetchPlan{}; + EXPECT_TRUE(client.PollPolicy().has_value()); + EXPECT_EQ(client.ExecutePrefetch(PrefetchPlan{}), ErrorCode::OK); + channel->send_ok = false; + EXPECT_EQ(client.ReportSnapshot(snapshot), ErrorCode::RPC_FAIL); + CfmClientImpl unavailable(nullptr); + EXPECT_EQ(unavailable.ReportSnapshot(snapshot), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); +} + +TEST(IoPatternFrameworkTest, CfmClientDispatchesReceivedPolicyCommands) { + auto channel = std::make_shared(); + int dispatched = 0; + CfmClientImpl client(channel, [&](const PolicyCommand& command) { + EXPECT_TRUE(std::holds_alternative(command)); + ++dispatched; + return ErrorCode::OK; + }); + + EXPECT_EQ(client.ReceivePolicy(PolicyCommand{PrefetchPlan{}}), ErrorCode::OK); + EXPECT_EQ(dispatched, 1); + channel->policy = PolicyCommand{AdmissionResult{}}; + EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); + EXPECT_EQ(dispatched, 2); +} + +TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { + auto flaky = std::make_shared(); + flaky->send_failures = 2; + ResilientCfmChannel channel(flaky, CfmRetryConfig{.max_retries = 2, + .degrade_after_failures = 2}); + EXPECT_TRUE(channel.SendSnapshot({})); + EXPECT_FALSE(channel.degraded()); + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_FAIL); + EXPECT_FALSE(channel.degraded()); + EXPECT_EQ(channel.consecutive_failures(), 1); + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_FAIL); + EXPECT_EQ(channel.consecutive_failures(), 2); + EXPECT_TRUE(channel.degraded()); +} + +TEST(IoPatternFrameworkTest, ResilientAnalyzerFallsBackAfterFailure) { + ResilientAnalyzer analyzer(std::make_shared(), 2); + EXPECT_EQ(analyzer.DetectWorkloadType({}), WorkloadType::kMixed); + EXPECT_FALSE(analyzer.degraded()); + EXPECT_EQ(analyzer.DetectWorkloadType({}), WorkloadType::kMixed); + EXPECT_TRUE(analyzer.degraded()); + EXPECT_EQ(analyzer.failures(), 2); +} + +TEST(IoPatternFrameworkTest, RpcChannelUsesCodecTransportAndTimeout) { + auto transport = std::make_shared(); + auto codec = std::make_shared(); + CfmRpcChannel channel(transport, codec, CfmRpcConfig{.timeout = std::chrono::milliseconds(25)}); + EXPECT_TRUE(channel.SendSnapshot({})); + EXPECT_EQ(transport->last_method, "report_snapshot"); + EXPECT_EQ(transport->last_payload, "snapshot"); + EXPECT_EQ(transport->last_timeout, std::chrono::milliseconds(25)); + transport->response = "policy"; + EXPECT_TRUE(channel.PollPolicy().has_value()); + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::OK); + auto rpc_channel = std::make_shared(transport, codec); + IoPatternReporter reporter(2, MakeCfmMetricBatchSink(rpc_channel)); + reporter.Enqueue(InferenceMetrics{}); + EXPECT_TRUE(reporter.Flush()); + EXPECT_EQ(transport->last_method, "report_metric_batch"); + EXPECT_EQ(transport->last_payload, "batch"); + transport->send_ok = false; + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_TIMEOUT); +} + +TEST(IoPatternFrameworkTest, BinaryCfmCodecRoundTripsAllPolicyCommands) { + CfmBinaryCodec codec; + PrefetchPlan prefetch{.strategy = PrefetchStrategy::kTimeout, + .timeout_us = 42, + .candidates = {PrefetchCandidate{ + .object = {TenantId("tenant-a"), "key"}, + .source_tier = CacheTier::kL3NofSsd, + .target_tier = CacheTier::kL2Segment, + .bytes = 512, + .priority = 0.8F, + .confidence = 0.9F}}}; + const auto decoded_prefetch = codec.DecodePolicy(codec.EncodePolicy(prefetch)); + ASSERT_TRUE(decoded_prefetch.has_value()); + const auto& decoded_plan = std::get(*decoded_prefetch); + ASSERT_EQ(decoded_plan.candidates.size(), 1); + EXPECT_EQ(decoded_plan.candidates.front().object.key, "key"); + EXPECT_EQ(decoded_plan.timeout_us, 42); + + AdmissionResult admission{.object = {TenantId("tenant-b"), "admit"}, + .target_tier = CacheTier::kL1Host, + .decision = AdmissionDecision::kAdmit, + .confidence = 0.75F}; + const auto decoded_admission = codec.DecodePolicy(codec.EncodePolicy(admission)); + ASSERT_TRUE(decoded_admission.has_value()); + EXPECT_EQ(std::get(*decoded_admission).object.key, "admit"); + + EvictionPlan eviction{.source_tier = CacheTier::kL1Host, + .target_bytes = 128, + .candidates = {EvictionCandidate{ + .object = {TenantId("tenant-c"), "evict"}, + .bytes = 128, + .score = 0.4F}}}; + const auto decoded_eviction = codec.DecodePolicy(codec.EncodePolicy(eviction)); + ASSERT_TRUE(decoded_eviction.has_value()); + EXPECT_EQ(std::get(*decoded_eviction).candidates.front().object.key, + "evict"); + + IoPatternSnapshot snapshot{.generated_at_ns = 9, + .keys = {KeyMetrics{.object = {TenantId("tenant-d"), "full"}, + .session_id = "session", + .token_count = 16, + .active = true}}, + .storage = {StorageMetric{.source_id = "master", + .used_bytes = 42}}}; + const auto decoded_snapshot = codec.DecodeSnapshot(codec.EncodeSnapshot(snapshot)); + ASSERT_TRUE(decoded_snapshot.has_value()); + EXPECT_EQ(decoded_snapshot->keys.front().session_id, "session"); + EXPECT_EQ(decoded_snapshot->storage.front().used_bytes, 42); + + MetricBatch batch{.inference = {InferenceMetrics{.object = {TenantId("tenant"), "metric"}, + .session_id = "s"}}, + .accesses = {AccessRecord{.object = {TenantId("tenant"), "metric"}, + .is_hit = true}}}; + const auto decoded_batch = codec.DecodeMetricBatch(codec.EncodeMetricBatch(batch)); + ASSERT_TRUE(decoded_batch.has_value()); + EXPECT_EQ(decoded_batch->inference.front().session_id, "s"); + EXPECT_TRUE(decoded_batch->accesses.front().is_hit); +} + +TEST(IoPatternFrameworkTest, InProcessCfmTransportAuthenticatesAndDispatches) { + CfmBinaryCodec codec; + bool received_snapshot = false; + auto transport = std::make_shared( + "shared-secret", [&received_snapshot](std::string_view method, + std::string_view) { + received_snapshot = method == "report_snapshot"; + return received_snapshot; + }); + CfmRpcChannel authorized(transport, std::make_shared(), + {.auth_token = "shared-secret"}); + EXPECT_TRUE(authorized.SendSnapshot({})); + EXPECT_TRUE(received_snapshot); + + transport->EnqueuePolicy(codec.EncodePolicy( + AdmissionResult{.object = {TenantId("tenant"), "key"}, + .decision = AdmissionDecision::kAdmit})); + ASSERT_TRUE(authorized.PollPolicy().has_value()); + + auto rejected = std::make_shared("secret"); + CfmRpcChannel unauthorized(rejected, std::make_shared(), + {.auth_token = "wrong"}); + EXPECT_FALSE(unauthorized.SendSnapshot({})); +} + +TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{.eviction = [](const EvictionPlan&) { + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { + return ErrorCode::OK; + }, + .admission = [](const AdmissionResult&) { + return ErrorCode::OK; + }}); + auto codec = std::make_shared(); + CfmIngress ingress(runtime, codec); + MetricBatch batch{.inference = {InferenceMetrics{ + .object = {TenantId("tenant"), "metric-key"}, + .session_id = "session", + .token_count = 32}}, + .accesses = {AccessRecord{ + .object = {TenantId("tenant"), "metric-key"}, + .block_size = 64, + .is_hit = true}}}; + EXPECT_TRUE(ingress.Handle("report_metric_batch", codec->EncodeMetricBatch(batch))); + const auto snapshot = runtime->Snapshot(); + ASSERT_EQ(snapshot.keys.size(), 1); + EXPECT_EQ(snapshot.keys.front().session_id, "session"); + EXPECT_EQ(snapshot.keys.front().access_count_window, 1); +} + +TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { + size_t batches = 0; + IoPatternReporter reporter(4, [&](const MetricBatch&) { + ++batches; + return true; + }); + reporter.Enqueue(InferenceMetrics{}); + reporter.Start(); + reporter.Stop(); + EXPECT_EQ(batches, 1); +} + +TEST(IoPatternFrameworkTest, DegradingPolicyEngineSwitchesToFallback) { + auto primary = std::make_shared( + std::make_shared(), nullptr, nullptr); + auto fallback = std::make_shared(nullptr, nullptr, + nullptr); + DegradingPolicyEngine engine(primary, fallback, 2); + EXPECT_FALSE(engine.degraded()); + EXPECT_EQ(engine.PlanEviction({}, CacheTier::kL1Host, 10).target_bytes, 10); + engine.RecordFailure(); + engine.RecordFailure(); + EXPECT_TRUE(engine.degraded()); + EXPECT_TRUE(engine.PlanEviction({}, CacheTier::kL1Host, 10) + .candidates.empty()); + engine.ForceDegraded(false); + EXPECT_FALSE(engine.degraded()); + EXPECT_EQ(engine.consecutive_failures(), 0); +} + +TEST(IoPatternFrameworkTest, FeedbackWindowAggregatesBoundedSamples) { + PolicyFeedbackWindow window(2); + window.Record({.hit_rate_delta = -0.2F, .prefetch_accuracy = 0.5F}); + window.Record({.hit_rate_delta = 0.1F, .prefetch_accuracy = 0.9F}); + window.Record({.hit_rate_delta = -0.4F, .prefetch_accuracy = 0.3F}); + const auto stats = window.Snapshot(); + EXPECT_EQ(stats.samples, 2); + EXPECT_FLOAT_EQ(stats.hit_rate_delta, (-0.2F - 0.4F) / 2.0F); + EXPECT_FLOAT_EQ(stats.prefetch_accuracy, (0.9F + 0.3F) / 2.0F); +} + +TEST(IoPatternFrameworkTest, AdaptiveTunerChangesWeightsAfterNegativeStreak) { + AdaptivePolicyTuner tuner(3); + ScoreBasedEvictionConfig config; + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + EXPECT_TRUE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + EXPECT_FLOAT_EQ(config.frequency_weight, 0.8F); + EXPECT_FLOAT_EQ(config.idle_weight, 1.1F); + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = 0.0F}, config)); +} + +TEST(IoPatternFrameworkTest, AdaptiveTunerHandlesChurnAndPersistsChanges) { + AdaptivePolicyTuner tuner(3); + ScoreBasedEvictionConfig config; + bool persisted = false; + tuner.SetPersistenceCallback( + [&](const ScoreBasedEvictionConfig&) { persisted = true; }); + EXPECT_TRUE(tuner.Tune({.eviction_churn = 0.8F}, config)); + EXPECT_TRUE(tuner.conservative()); + EXPECT_TRUE(persisted); +} + +TEST(IoPatternFrameworkTest, ObservabilityTracksPolicyAndDegradeCounters) { + IoPatternObservability metrics; + metrics.RecordCollectLatency(10); + metrics.RecordCollectLatency(3); + metrics.RecordAnalyzeLatency(20); + metrics.RecordPolicyDecision(true); + metrics.RecordPolicyDecision(false); + metrics.RecordFalsePositive(); + metrics.RecordDegrade(); + metrics.RecordReportDrop(2); + const auto snapshot = metrics.Snapshot(); + EXPECT_EQ(snapshot.collect_latency_us, 10); + EXPECT_EQ(snapshot.analyze_latency_us, 20); + EXPECT_EQ(snapshot.policy_decisions, 2); + EXPECT_EQ(snapshot.strategy_hits, 1); + EXPECT_EQ(snapshot.strategy_trials, 2); + EXPECT_EQ(snapshot.false_positives, 1); + EXPECT_EQ(snapshot.degrade_count, 1); + EXPECT_EQ(snapshot.report_drop_count, 2); + EXPECT_FLOAT_EQ(snapshot.strategy_hit_rate, 0.5F); + EXPECT_FLOAT_EQ(snapshot.false_positive_rate, 0.5F); + EXPECT_FLOAT_EQ(metrics.Snapshot(2.0).policy_decision_qps, 1.0F); +} + +TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { + SlidingWindowAnalyzer analyzer(100); + IoPatternSnapshot first; + first.generated_at_ns = 10; + first.keys.push_back(KeyMetrics{.token_count = 20 * 1024, + .prefix_fanout = 20, + .match_length = 512, + .block_size = 100, + .access_count_window = 1}); + IoPatternSnapshot second; + second.generated_at_ns = 50; + second.keys.push_back(KeyMetrics{.token_count = 30, + .prefix_fanout = 20, + .match_length = 300, + .block_size = 300, + .access_count_window = 5}); + EXPECT_EQ(analyzer.DetectWorkloadType(second), WorkloadType::kMixed); + const auto stats = analyzer.FeatureStats(); + EXPECT_EQ(stats.samples, 2); + EXPECT_EQ(stats.token_median, 30); + EXPECT_EQ(stats.fanout_p90, 20); + EXPECT_EQ(stats.block_p90, 300); +} + +TEST(IoPatternFrameworkTest, KMeansFallbackLabelsIndependentSessions) { + SlidingWindowAnalyzer analyzer; + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = 1; + snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "long"}, + .session_id = "code-session", + .token_count = 20 * 1024, + .prefix_fanout = 20, + .match_length = 512}, + KeyMetrics{.object = {TenantId("tenant-b"), "small"}, + .session_id = "recommendation-session", + .block_size = 64 * 1024, + .access_count_window = 30}, + }; + + const auto result = analyzer.Analyze(snapshot); + EXPECT_EQ(result.workload_type, WorkloadType::kMixed); + ASSERT_EQ(result.sessions.size(), 2); + EXPECT_NE(result.sessions[0].workload_type, result.sessions[1].workload_type); +} + +TEST(IoPatternFrameworkTest, TierExecutorBridgesPolicyResults) { + int evictions = 0; + int prefetches = 0; + int admissions = 0; + TierOperationExecutor executor( + [&](const EvictionPlan&) { ++evictions; return ErrorCode::OK; }, + [&](const PrefetchPlan&) { ++prefetches; return ErrorCode::OK; }, + [&](const AdmissionResult&) { ++admissions; return ErrorCode::OK; }); + PolicyResult result; + result.eviction.target_bytes = 1024; + result.prefetch.candidates.push_back(PrefetchCandidate{}); + result.admissions.push_back( + AdmissionResult{.decision = AdmissionDecision::kAdmit}); + const auto status = executor.Execute(result); + EXPECT_EQ(status.eviction, ErrorCode::OK); + EXPECT_EQ(status.prefetch, ErrorCode::OK); + ASSERT_EQ(status.admissions.size(), 1); + EXPECT_EQ(status.admissions.front(), ErrorCode::OK); + EXPECT_EQ(evictions, 1); + EXPECT_EQ(prefetches, 1); + EXPECT_EQ(admissions, 1); + + TierOperationExecutor degraded({}, {}, {}); + const auto degraded_status = degraded.Execute(result); + EXPECT_TRUE(degraded_status.degraded); + EXPECT_EQ(degraded_status.prefetch, + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); +} + +TEST(IoPatternFrameworkTest, LegacyEvictionAdapterUsesLruFallback) { + auto lru = std::make_shared(); + LegacyEvictionOps fallback(lru); + PolicyContext context; + KeyMetrics first{.object = {TenantId("tenant-a"), "first"}, + .block_size = 10, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}; + KeyMetrics second{.object = {TenantId("tenant-a"), "second"}, + .block_size = 20, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}; + context.snapshot.keys = {first, second}; + const auto plan = fallback.Evaluate(context, CacheTier::kL1Host, 10); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().object.key, "first"); +} + +TEST(IoPatternFrameworkTest, ScoreBasedEvictionUsesTierSpecificSignals) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "small-single-copy"}, + .block_size = 10, + .other_replica_count = 0, + .replica_tiers = CacheTierBit(CacheTier::kL3NofSsd)}, + KeyMetrics{.object = {TenantId("tenant-a"), "large-redundant"}, + .block_size = 100, + .other_replica_count = 1, + .replica_tiers = CacheTierBit(CacheTier::kL3NofSsd)}, + }; + context.analysis.keys = { + KeyPattern{.object = context.snapshot.keys[0].object, .idle_score = 1.0F}, + KeyPattern{.object = context.snapshot.keys[1].object, .idle_score = 1.0F}, + }; + + ScoreBasedEvictionOps eviction; + const auto plan = eviction.Evaluate(context, CacheTier::kL3NofSsd, 110); + + ASSERT_EQ(plan.candidates.size(), 2); + EXPECT_EQ(plan.candidates.front().object.key, "large-redundant"); + EXPECT_EQ(plan.candidates.front().target_tier, CacheTier::kL3NofSsd); +} + +TEST(IoPatternFrameworkTest, TierDownTemplatesChooseDocumentedTargets) { + PolicyContext context; + context.snapshot.keys = {KeyMetrics{ + .object = {TenantId("tenant"), "key"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL0Hbm)}}; + context.analysis.keys = {KeyPattern{.object = context.snapshot.keys.front().object}}; + + ScoreBasedEvictionOps code({.tier_down_mode = TierDownMode::kSkipHost}); + EXPECT_EQ(code.Evaluate(context, CacheTier::kL0Hbm, 64) + .candidates.front().target_tier, + CacheTier::kL2Segment); + ScoreBasedEvictionOps recommendation( + {.tier_down_mode = TierDownMode::kStepwise}); + EXPECT_EQ(recommendation.Evaluate(context, CacheTier::kL0Hbm, 64) + .candidates.front().target_tier, + CacheTier::kL1Host); +} + +TEST(IoPatternFrameworkTest, PrefetchRequiresConfidenceAndNeverPromotesToHbm) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "low-confidence"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL3NofSsd)}, + KeyMetrics{.object = {TenantId("tenant-a"), "host-only"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}, + }; + context.analysis.keys = { + KeyPattern{.object = context.snapshot.keys[0].object, .confidence = 0.5F}, + KeyPattern{.object = context.snapshot.keys[1].object, .confidence = 0.9F}, + }; + TraceHistory trace{.events = { + TraceEvent{.object = context.snapshot.keys[0].object, + .match_length = 512, + .is_hit = true}, + TraceEvent{.object = context.snapshot.keys[1].object, + .match_length = 512, + .is_hit = true}, + }}; + + TraceBasedPrefetchOps prefetch; + const auto plan = prefetch.Evaluate(context, trace); + + EXPECT_TRUE(plan.candidates.empty()); +} + +TEST(IoPatternFrameworkTest, RuntimeConnectsCollectionAnalysisPolicyAndHandlers) { + int evictions = 0; + int prefetches = 0; + int admissions = 0; + IoPatternRuntime runtime( + IoPatternRuntime::Handlers{ + .eviction = [&](const EvictionPlan&) { + ++evictions; + return ErrorCode::OK; + }, + .prefetch = [&](const PrefetchPlan&) { + ++prefetches; + return ErrorCode::OK; + }, + .admission = [&](const AdmissionResult&) { + ++admissions; + return ErrorCode::OK; + }, + }); + + AccessRecord access{.object = {TenantId("tenant-a"), "runtime-key"}, + .observed_at_ns = 1, + .block_size = 64, + .tier = CacheTier::kL2Segment, + .is_hit = true}; + runtime.RecordAccess(access.object.key, access); + runtime.ReportInferenceMetrics( + InferenceMetrics{.object = access.object, .match_length = 512}); + + const auto status = runtime.Execute(CacheTier::kL2Segment, 64, + TraceHistory{}, {access.object}); + + EXPECT_EQ(status.eviction, ErrorCode::OK); + ASSERT_EQ(status.admissions.size(), 1); + EXPECT_EQ(status.admissions.front(), ErrorCode::OK); + EXPECT_EQ(evictions, 1); + EXPECT_EQ(admissions, 1); + EXPECT_GE(runtime.Snapshot().keys.size(), 1); +} + +TEST(IoPatternFrameworkTest, RuntimeExecutesCfmCommandsThroughStorageHandlers) { + int admissions = 0; + IoPatternRuntime runtime( + {.eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [&admissions](const AdmissionResult&) { + ++admissions; + return ErrorCode::OK; + }}); + CfmClientImpl client( + std::make_shared(), + [&runtime](const PolicyCommand& command) { + return runtime.ExecuteCommand(command); + }); + EXPECT_EQ(client.ReceivePolicy( + AdmissionResult{.object = {TenantId("tenant"), "key"}, + .decision = AdmissionDecision::kAdmit}), + ErrorCode::OK); + EXPECT_EQ(admissions, 1); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-wheel/mooncake/io_pattern_bridge.py b/mooncake-wheel/mooncake/io_pattern_bridge.py new file mode 100644 index 0000000000..cd0c46d729 --- /dev/null +++ b/mooncake-wheel/mooncake/io_pattern_bridge.py @@ -0,0 +1,81 @@ +"""Framework-neutral, non-blocking CFM metric bridges. + +The vLLM connector accepts these objects through ``vllm_config``. SGLang's +HiCache integration can instantiate :class:`SglangHiCacheIoPatternBridge` at +its request-finished and prefix-match hooks without depending on vLLM. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from queue import Empty, Full, Queue +from threading import Event, Thread +from typing import Any + + +MetricSink = Callable[[Mapping[str, Any]], None] + + +class BatchedIoPatternBridge: + """Bounded asynchronous bridge to a CFM metric reporter. + + ``report`` receives complete records (for example, a CFM RPC client + method). Back pressure drops metrics instead of delaying inference. + """ + + def __init__(self, report: MetricSink, capacity: int = 4096) -> None: + self._report = report + self._queue: Queue[dict[str, Any]] = Queue(maxsize=capacity) + self._stopping = Event() + self._worker = Thread(target=self._run, name="io-pattern-cfm", + daemon=True) + self._worker.start() + self.dropped = 0 + + def report_inference_metrics(self, **metrics: Any) -> None: + try: + self._queue.put_nowait(dict(metrics)) + except Full: + self.dropped += 1 + + def close(self) -> None: + self._stopping.set() + self._worker.join(timeout=1.0) + + def _run(self) -> None: + while not self._stopping.is_set() or not self._queue.empty(): + try: + metrics = self._queue.get(timeout=0.1) + except Empty: + continue + try: + self._report(metrics) + except Exception: + # A failed CFM report must remain isolated from inference. + self.dropped += 1 + + +class SglangHiCacheIoPatternBridge(BatchedIoPatternBridge): + """Adapter for SGLang HiCache request and prefix-match hooks. + + ``layout`` must be the active ``--hicache-mem-layout`` value, normally + ``layer_first``, ``page_first`` or ``page_first_direct``. + """ + + def request_finished(self, *, session_id: str, token_count: int, + prefix_depth: int, prefix_fanout: int, + match_length: int, continuous_prefix_length: int, + recompute_cost: float, request_priority: int = 0, + layout: str = "layer_first", layout_group: int = 0) -> None: + self.report_inference_metrics( + session_id=session_id, + token_count=token_count, + prefix_depth=prefix_depth, + prefix_fanout=prefix_fanout, + match_length=match_length, + continuous_prefix_length=continuous_prefix_length, + recompute_cost=recompute_cost, + request_priority=request_priority, + layout=layout, + layout_group=layout_group, + ) diff --git a/mooncake-wheel/mooncake/mooncake_connector_v1.py b/mooncake-wheel/mooncake/mooncake_connector_v1.py index 18873b38fc..79a9b721ae 100644 --- a/mooncake-wheel/mooncake/mooncake_connector_v1.py +++ b/mooncake-wheel/mooncake/mooncake_connector_v1.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from queue import Queue from os import getenv -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional import msgspec import numpy as np @@ -71,6 +71,31 @@ logger = init_logger(__name__) +class IoPatternBridge: + """Optional connector-side bridge for IO Pattern metric reporting. + + Deployments may attach an object implementing ``report_inference_metrics`` + to ``vllm_config``. The connector remains usable when it is absent. + """ + + def report_inference_metrics(self, **metrics: Any) -> None: + raise NotImplementedError + + +class CallbackIoPatternBridge(IoPatternBridge): + """Concrete bridge that forwards complete metric records to a CFM adapter. + + The callback is deliberately injected by the deployment so this connector + stays independent of a particular Python/C++ RPC binding. + """ + + def __init__(self, report: Callable[..., None]) -> None: + self._report = report + + def report_inference_metrics(self, **metrics: Any) -> None: + self._report(**metrics) + + class MooncakeAgentMetadata( msgspec.Struct, omit_defaults=True, # type: ignore[call-arg] @@ -131,6 +156,8 @@ def __init__(self, vllm_config: VllmConfig, role: KVConnectorRole): assert vllm_config.kv_transfer_config.engine_id is not None super().__init__(vllm_config, role) self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id + self.io_pattern_bridge: Optional[IoPatternBridge] = getattr( + vllm_config, "io_pattern_bridge", None) if role == KVConnectorRole.SCHEDULER: self.connector_scheduler: Optional[MooncakeConnectorScheduler] = \ @@ -235,6 +262,12 @@ class MooncakeConnectorScheduler: def __init__(self, vllm_config: VllmConfig, engine_id: str): self.vllm_config = vllm_config self.engine_id: EngineId = engine_id + self.io_pattern_bridge: Optional[IoPatternBridge] = getattr( + vllm_config, "io_pattern_bridge", None) + # Kept on the scheduler, which owns all three hook points below. + # Each request contributes partial observations until completion. + self._io_pattern_metrics: dict[ReqId, dict[str, Any]] = {} + self.io_pattern_layout = get_kv_cache_layout() self.side_channel_host = get_ip() self.side_channel_port = get_mooncake_side_channel_port(vllm_config) @@ -277,6 +310,11 @@ def get_num_new_matched_tokens( # Remote prefill: get all prompt blocks from remote. count = len(request.prompt_token_ids) - num_computed_tokens if count > 0: + self._io_pattern_metrics.setdefault(request.request_id, {}).update( + match_length=count, + continuous_prefix_length=count, + token_count=len(request.prompt_token_ids), + ) return count, True # No remote prefill for this request. @@ -295,6 +333,12 @@ def update_state_after_alloc(self, request: "Request", if not params: return + self._io_pattern_metrics.setdefault(request.request_id, {}).update( + prefix_depth=params.get("prefix_depth", len(blocks.get_unhashed_block_ids())), + prefix_fanout=params.get("prefix_fanout", 0), + continuous_prefix_length=num_external_tokens, + ) + if params.get("do_remote_prefill"): assert self.kv_role != "kv_producer" if all(p in params for p in ("remote_host", "remote_port")): @@ -357,7 +401,29 @@ def request_finished( "MooncakeConnector request_finished, request_status=%s, " "kv_transfer_params=%s", request.status, params) if not params: + # A request may finish before allocation; discard any partial + # observation so aborted requests cannot accumulate indefinitely. + self._io_pattern_metrics.pop(request.request_id, None) return False, None + if self.io_pattern_bridge is not None: + try: + metrics = self._io_pattern_metrics.pop(request.request_id, {}) + token_count = metrics.get("token_count", len(block_ids)) + self.io_pattern_bridge.report_inference_metrics( + session_id=request.request_id, + token_count=token_count, + prefix_depth=metrics.get("prefix_depth", params.get("prefix_depth", 0)), + prefix_fanout=metrics.get("prefix_fanout", params.get("prefix_fanout", 0)), + match_length=metrics.get("match_length", params.get("match_length", 0)), + continuous_prefix_length=metrics.get( + "continuous_prefix_length", params.get("continuous_prefix_length", 0)), + recompute_cost=params.get("recompute_cost", float(token_count)), + request_priority=getattr(request, "priority", 0), + layout=self.io_pattern_layout, + layout_group=params.get("layout_group", 0), + ) + except Exception: # metrics must never affect the data path + logger.debug("IO Pattern metric report failed", exc_info=True) if params.get("do_remote_prefill"): # If do_remote_prefill is still True when the request is finished, From 62519213090f0e8e713479f35c6adc7e17948264 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 2 Sep 2026 15:38:53 +0800 Subject: [PATCH 4/4] =?UTF-8?q?io=20pattern=E4=BB=A3=E7=A0=81=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E6=AD=A5=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 71 +++- .../include/io_pattern/cfm_channel.h | 30 +- .../include/io_pattern/cfm_ingress.h | 3 +- .../include/io_pattern/cfm_service.h | 91 ++++ .../include/io_pattern/collector_impl.h | 22 +- .../include/io_pattern/io_pattern.h | 1 + mooncake-store/include/io_pattern/reporter.h | 4 + .../io_pattern/resilient_cfm_channel.h | 3 +- .../include/io_pattern/rpc_transport.h | 58 ++- mooncake-store/include/io_pattern/runtime.h | 41 +- .../io_pattern/sliding_window_analyzer.h | 6 +- mooncake-store/include/master_config.h | 29 ++ mooncake-store/include/master_service.h | 24 +- mooncake-store/include/rpc_service.h | 4 + mooncake-store/src/CMakeLists.txt | 1 + .../src/io_pattern/cfm_client_impl.cpp | 14 +- mooncake-store/src/io_pattern/cfm_ingress.cpp | 27 +- mooncake-store/src/io_pattern/cfm_service.cpp | 284 +++++++++++++ .../src/io_pattern/collector_impl.cpp | 118 +++++- .../src/io_pattern/policy_strategies.cpp | 21 +- mooncake-store/src/io_pattern/reporter.cpp | 32 +- .../src/io_pattern/resilient_cfm_channel.cpp | 21 +- .../src/io_pattern/rpc_transport.cpp | 213 +++++++++- mooncake-store/src/io_pattern/runtime.cpp | 146 +++++-- .../io_pattern/sliding_window_analyzer.cpp | 68 ++- mooncake-store/src/master.cpp | 99 +++++ mooncake-store/src/master_service.cpp | 197 ++++++++- mooncake-store/src/rpc_service.cpp | 9 +- .../tests/io_pattern_framework_test.cpp | 399 +++++++++++++++++- .../tests/master_service_config_test.cpp | 22 + 30 files changed, 1901 insertions(+), 157 deletions(-) create mode 100644 mooncake-store/include/io_pattern/cfm_service.h create mode 100644 mooncake-store/src/io_pattern/cfm_service.cpp diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 1c9ab5818c..59a7a656bf 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -982,7 +982,7 @@ by the Store master. - `IoPatternReporter` provides bounded, non-blocking batches with explicit report/drop counters and a transport-agnostic sink. - `MetricBatchTransport` defines the transport seam, and the reporter exposes - load-sensitive 100/500/1000 ms flush recommendations. + load-sensitive 100/200/500/1000 ms flush recommendations. - `IoPatternRuntime` wires collection, bounded analysis, policy execution, feedback tuning and storage handlers; `MasterService` feeds it from actual Get/Put/watermark paths. @@ -1034,6 +1034,24 @@ by the Store master. - Analyzer execution has a single in-flight worker, timeout fallback to the last safe result, and an explicit key-count budget; collector key quotas and reporter bounds provide the associated overload/OOM protection. +- Access and write frequencies use timestamped buckets pruned against a true + rolling 60-second cutoff. Sliding analysis deduplicates objects across + snapshots and enforces a hard total retained-key budget (including a single + oversized snapshot), so repeated high-watermark evaluations cannot multiply + complete snapshots without bound. CFM ingress rebases process-local monotonic + timestamps to receiver time, and each object also has a hard bucket-count cap. +- Store-side eviction executes only the tenant-qualified objects selected by + the policy. The legacy `BatchEvict` path runs only when policy execution + fails, avoiding a second unplanned eviction pass. +- Non-memory PUT completions enqueue bounded, asynchronous L1 retention/ + promotion evaluation. This is a post-write cache-admission hook, not initial + replica placement: the existing `PutStart` contract selects and allocates + replicas before write metrics such as batch and overwrite are known. +- CFM polling distinguishes a command, a healthy empty queue and a transport + error. Only transport errors contribute to consecutive-failure degradation. +- Reporter intervals follow the documented memory/RPC load thresholds + (100/200/500/1000 ms), and in-process transport callbacks execute outside the + transport mutex. ## Interface decision: complete plans versus document shorthand @@ -1088,10 +1106,47 @@ Registry ownership is external and thread-safe. Factories return independent Ops instances; callers own the returned smart pointers. Concrete storage and RPC resources are injected through execution handlers and CFM channels. -## Known gaps - -There are no remaining implementation gaps in the Mooncake IO Pattern scope. -Production deployments select their network-specific `CfmRpcTransport` through -the documented transport seam; the authenticated embedded transport is the -reference implementation and the SGLang adapter is intentionally kept -framework-neutral because SGLang source is not vendored in this repository. +## Production CFM wiring + +Master registers authenticated CFM handlers on its existing `coro_rpc` port. +`CoroRpcCfmTransport` is the production client: metric batches are delivered to +`CfmIngress`, while policy commands use a bounded per-node queue and are polled +by stable `node_id`. Received commands execute through +`IoPatternRuntime::ExecuteCommand`, preserving the same storage-safe handlers as +local policy decisions. Every report RPC also carries that `node_id`; ingress +uses it as the authoritative storage-metric source so central aggregation does +not merge watermarks from different Masters. + +Configure a central CFM receiver with `io_pattern_cfm_auth_token`. Configure each +reporting/policy-consuming Master with: + +- `io_pattern_cfm_endpoint=host:port` +- `io_pattern_cfm_node_id=` (defaults to `cluster_id`) +- the same `io_pattern_cfm_auth_token` +- on the central receiver only, a distinct + `io_pattern_cfm_producer_auth_token` for policy producers +- optional `io_pattern_cfm_timeout_ms` and + `io_pattern_cfm_policy_queue_capacity` + +An outbound Master authenticates during construction and fails startup if the +configured CFM endpoint cannot be reached or rejects the token. At runtime the +Reporter sends metric batches over the channel and a resilient poll loop +dispatches queued policies. On the receiver, each accepted metric batch is put +onto a bounded policy-production queue; the central runtime runs +Collector -> Analyzer -> PolicyEngine asynchronously and automatically queues +high-watermark eviction and trace-derived prefetch commands for the reporting +`node_id`. An external policy producer may also call the registered +`CfmRpcService::EnqueuePolicy` RPC with a target node id and an encoded +`PolicyCommand`. + +Node credentials cannot use that explicit enqueue RPC; an external producer +must authenticate with the separately configured producer credential. The +server validates commands before enqueueing them, assigns a delivery id, and +retains each command until the target node acknowledges successful execution. +Its poll response distinguishes an authenticated empty queue from a rejected +or invalid request, so authorization failures enter the normal degradation +path. The configured capacity is enforced for both pending production work and +policy delivery, with policy delivery bounded both per node and globally. + +The SGLang adapter remains framework-neutral because SGLang source is not +vendored in this repository. diff --git a/mooncake-store/include/io_pattern/cfm_channel.h b/mooncake-store/include/io_pattern/cfm_channel.h index 5533e113f0..7d0b84e125 100644 --- a/mooncake-store/include/io_pattern/cfm_channel.h +++ b/mooncake-store/include/io_pattern/cfm_channel.h @@ -1,19 +1,47 @@ #pragma once #include +#include #include "types.h" #include "../types.h" namespace mooncake::io_pattern { +struct CfmPollResult { + enum class Status { kCommand, kEmpty, kError }; + + static CfmPollResult Command(PolicyCommand command, + uint64_t delivery_id = 0) { + return {.status = Status::kCommand, + .command = std::move(command), + .delivery_id = delivery_id}; + } + static CfmPollResult Empty() { return {.status = Status::kEmpty}; } + static CfmPollResult Error() { return {.status = Status::kError}; } + + Status status{Status::kEmpty}; + std::optional command; + uint64_t delivery_id{0}; +}; + // Transport-neutral CFM RPC channel. Implementations own serialization, // retries and connection lifecycle. class CfmChannel { public: virtual ~CfmChannel() = default; virtual bool SendSnapshot(const IoPatternSnapshot& snapshot) = 0; - virtual std::optional PollPolicy() = 0; + virtual CfmPollResult PollPolicyResult() = 0; + std::optional PollPolicy() { + auto result = PollPolicyResult(); + if (result.status != CfmPollResult::Status::kCommand || + !result.command) { + return std::nullopt; + } + if (!AcknowledgePolicy(result.delivery_id, true)) return std::nullopt; + return std::move(result.command); + } + virtual bool AcknowledgePolicy(uint64_t, bool) { return true; } virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; }; diff --git a/mooncake-store/include/io_pattern/cfm_ingress.h b/mooncake-store/include/io_pattern/cfm_ingress.h index eb027e6575..f11a33079b 100644 --- a/mooncake-store/include/io_pattern/cfm_ingress.h +++ b/mooncake-store/include/io_pattern/cfm_ingress.h @@ -17,7 +17,8 @@ class CfmIngress final { std::make_shared()) : runtime_(std::move(runtime)), codec_(std::move(codec)) {} - bool Handle(std::string_view method, std::string_view payload); + bool Handle(std::string_view method, std::string_view payload, + std::string_view source_id = {}); private: std::shared_ptr runtime_; diff --git a/mooncake-store/include/io_pattern/cfm_service.h b/mooncake-store/include/io_pattern/cfm_service.h new file mode 100644 index 0000000000..40a53c2fab --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_service.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cfm_ingress.h" + +namespace mooncake::io_pattern { + +// Authenticated server-side CFM endpoint. The RPC layer delegates to this +// class, keeping authentication, bounded policy queues and runtime dispatch +// independent of the concrete network transport. +class CfmService final { + public: + CfmService(std::shared_ptr runtime, + std::string auth_token, size_t policy_queue_capacity = 4096, + std::string producer_auth_token = {}); + ~CfmService(); + + bool Authenticate(std::string_view token) const; + bool AuthenticateNode(std::string_view token) const; + bool AuthenticateProducer(std::string_view token) const; + bool Send(std::string_view node_id, std::string_view method, + std::string_view payload, std::string_view token); + std::optional> PollPolicy( + std::string_view node_id, std::string_view token); + bool AcknowledgePolicy(std::string_view node_id, uint64_t delivery_id, + bool success, std::string_view token); + bool EnqueuePolicy(std::string node_id, std::string payload, + std::string_view token); + + private: + bool EnqueueValidated(std::string node_id, std::string payload); + void SchedulePolicyProduction(std::string node_id, MetricBatch batch); + void PolicyProducerWorker(); + void ProducePolicies(std::string_view node_id, const MetricBatch& batch); + + std::shared_ptr runtime_; + std::shared_ptr codec_; + CfmIngress ingress_; + const std::string auth_token_; + const std::string producer_auth_token_; + const size_t policy_queue_capacity_; + std::mutex mutex_; + std::unordered_map>> + policy_queues_; + uint64_t next_delivery_id_{1}; + size_t total_queued_policies_{0}; + std::mutex producer_mutex_; + std::condition_variable producer_cv_; + std::deque> pending_metric_batches_; + bool producer_stopping_{false}; + std::thread producer_worker_; +}; + +// coro_rpc-facing adapter. Keeping RPC signatures here lets both the Master +// server and integration tests register the exact production endpoints. +class CfmRpcService final { + public: + explicit CfmRpcService(std::shared_ptr service) + : service_(std::move(service)) {} + + bool Authenticate(const std::string& auth_token); + bool Send(const std::string& node_id, const std::string& method, + const std::string& payload, const std::string& auth_token); + // The boolean explicitly distinguishes a rejected request from an + // authenticated queue that currently has no policy. + std::pair>> Receive( + const std::string& method, const std::string& node_id, + const std::string& auth_token); + bool Acknowledge(const std::string& node_id, uint64_t delivery_id, + bool success, const std::string& auth_token); + bool EnqueuePolicy(const std::string& node_id, const std::string& payload, + const std::string& auth_token); + + private: + std::shared_ptr service_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector_impl.h b/mooncake-store/include/io_pattern/collector_impl.h index 4324360750..fb2a43844d 100644 --- a/mooncake-store/include/io_pattern/collector_impl.h +++ b/mooncake-store/include/io_pattern/collector_impl.h @@ -1,8 +1,10 @@ #pragma once +#include +#include +#include #include #include -#include #include #include "collector.h" @@ -17,6 +19,10 @@ class IoPatternCollectorImpl final : public IoPatternCollector { struct Config { size_t max_keys_per_tenant{0}; size_t max_total_keys{0}; + uint64_t access_window_ns{60'000'000'000ULL}; + uint64_t access_bucket_ns{1'000'000'000ULL}; + size_t max_access_buckets_per_key{64}; + std::function now_ns; }; explicit IoPatternCollectorImpl(Config config = {}, @@ -47,6 +53,16 @@ class IoPatternCollectorImpl final : public IoPatternCollector { (static_cast(key.tier) << 1); } }; + struct AccessWindowBucket { + uint64_t observed_at_ns{0}; + uint64_t access_count{0}; + uint64_t write_count{0}; + uint64_t overwrite_count{0}; + uint32_t max_write_batch_size{0}; + }; + + void ApplyAccessWindow(const ObjectRef& object, uint64_t now_ns, + KeyMetrics& metrics) const; mutable std::mutex mutex_; Config config_; @@ -54,8 +70,8 @@ class IoPatternCollectorImpl final : public IoPatternCollector { uint64_t dropped_{0}; bool degraded_{false}; std::unordered_map key_metrics_; - std::unordered_map write_counts_; - std::unordered_map overwrite_counts_; + std::unordered_map, ObjectRefHash> + access_windows_; std::unordered_map tenant_key_counts_; std::unordered_map storage_metrics_; diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 0bc97def1d..78f3dbef1d 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -4,6 +4,7 @@ #include "io_pattern/client.h" #include "io_pattern/cfm_channel.h" #include "io_pattern/cfm_ingress.h" +#include "io_pattern/cfm_service.h" #include "io_pattern/cfm_protocol.h" #include "io_pattern/cfm_client_impl.h" #include "io_pattern/feedback.h" diff --git a/mooncake-store/include/io_pattern/reporter.h b/mooncake-store/include/io_pattern/reporter.h index fc0cbdf364..8926550fa5 100644 --- a/mooncake-store/include/io_pattern/reporter.h +++ b/mooncake-store/include/io_pattern/reporter.h @@ -46,11 +46,13 @@ class IoPatternReporter final { size_t pending() const; uint64_t dropped() const; uint64_t reported() const; + void UpdateLoad(float memory_used_ratio, uint64_t rpc_latency_us); std::chrono::milliseconds RecommendedFlushInterval() const; private: bool EnqueueImpl(std::function append, const TenantId& tenant); + std::chrono::milliseconds RecommendedFlushIntervalLocked() const; const size_t capacity_; const MetricBatchSink sink_; @@ -62,6 +64,8 @@ class IoPatternReporter final { std::condition_variable condition_; std::thread worker_; bool running_{false}; + float memory_used_ratio_{0.0F}; + uint64_t rpc_latency_us_{0}; std::unordered_map tenant_pending_; }; diff --git a/mooncake-store/include/io_pattern/resilient_cfm_channel.h b/mooncake-store/include/io_pattern/resilient_cfm_channel.h index d0b2118664..e022de2157 100644 --- a/mooncake-store/include/io_pattern/resilient_cfm_channel.h +++ b/mooncake-store/include/io_pattern/resilient_cfm_channel.h @@ -21,7 +21,8 @@ class ResilientCfmChannel final : public CfmChannel { : delegate_(std::move(delegate)), config_(config) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - std::optional PollPolicy() override; + CfmPollResult PollPolicyResult() override; + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; bool degraded() const; diff --git a/mooncake-store/include/io_pattern/rpc_transport.h b/mooncake-store/include/io_pattern/rpc_transport.h index 234f99c1ec..98ff83d830 100644 --- a/mooncake-store/include/io_pattern/rpc_transport.h +++ b/mooncake-store/include/io_pattern/rpc_transport.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "cfm_channel.h" @@ -16,6 +17,23 @@ namespace mooncake::io_pattern { +struct CfmReceiveResult { + enum class Status { kPayload, kEmpty, kError }; + + static CfmReceiveResult Payload(std::string payload, + uint64_t delivery_id = 0) { + return {.status = Status::kPayload, + .payload = std::move(payload), + .delivery_id = delivery_id}; + } + static CfmReceiveResult Empty() { return {.status = Status::kEmpty}; } + static CfmReceiveResult Error() { return {.status = Status::kError}; } + + Status status{Status::kEmpty}; + std::string payload; + uint64_t delivery_id{0}; +}; + class CfmRpcCodec { public: virtual ~CfmRpcCodec() = default; @@ -35,8 +53,34 @@ class CfmRpcTransport { virtual bool Authenticate(std::string_view token) { return token.empty(); } virtual bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) = 0; - virtual std::optional Receive( + virtual CfmReceiveResult Receive( std::string_view method, std::chrono::milliseconds timeout) = 0; + virtual bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds timeout) = 0; +}; + +// Production CFM transport over Mooncake's existing coro_rpc connection pool. +// It targets the CFM handlers registered on the Master RPC service. +class CoroRpcCfmTransport final : public CfmRpcTransport { + public: + CoroRpcCfmTransport(std::string endpoint, std::string node_id, + std::chrono::milliseconds default_timeout = + std::chrono::milliseconds(500)); + ~CoroRpcCfmTransport() override; + + bool Authenticate(std::string_view token) override; + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) override; + CfmReceiveResult Receive( + std::string_view method, std::chrono::milliseconds timeout) override; + bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds timeout) override; + bool EnqueuePolicy(std::string_view node_id, std::string_view payload, + std::chrono::milliseconds timeout); + + private: + class Impl; + std::unique_ptr impl_; }; struct CfmRpcConfig { @@ -58,8 +102,12 @@ class InProcessCfmRpcTransport final : public CfmRpcTransport { bool Authenticate(std::string_view token) override; bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) override; - std::optional Receive( + CfmReceiveResult Receive( std::string_view method, std::chrono::milliseconds timeout) override; + bool Acknowledge(uint64_t, bool, + std::chrono::milliseconds) override { + return true; + } void EnqueuePolicy(std::string payload); void SetSendHandler(SendHandler handler); @@ -82,7 +130,8 @@ class CfmRpcChannel final : public CfmChannel { config_(config) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - std::optional PollPolicy() override; + CfmPollResult PollPolicyResult() override; + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; bool SendMetricBatch(const MetricBatch& batch); @@ -105,7 +154,8 @@ class CfmChannelPool final : public CfmChannel { : channels_(std::move(channels)) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - std::optional PollPolicy() override; + CfmPollResult PollPolicyResult() override; + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; private: diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 74acfec1bd..7ec9fffc27 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -1,8 +1,12 @@ #pragma once #include +#include +#include #include #include +#include +#include #include #include @@ -39,6 +43,7 @@ class IoPatternRuntime final { size_t report_capacity{4096}; size_t report_per_tenant_capacity{0}; size_t max_pending_prefetches{4096}; + size_t max_pending_admissions{4096}; MetricBatchSink report_sink; LegacyFallback legacy_fallback{LegacyFallback::kLru}; }; @@ -56,9 +61,18 @@ class IoPatternRuntime final { const TraceHistory& trace, const std::vector& admissions = {}, const std::string& session_id = {}); + // Runs Collector -> Analyzer -> PolicyEngine without invoking local + // storage handlers. Central CFM uses this to produce commands for a + // target node; Store data paths continue to use Execute(). + PolicyResult Plan(CacheTier eviction_tier, uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions = {}, + const std::string& session_id = {}); // Applies a CFM-issued command through the same storage handlers as a // locally planned policy. This is the CFM-to-Store execution endpoint. ErrorCode ExecuteCommand(const PolicyCommand& command); + bool ScheduleAdmission(ObjectRef object, CacheTier target_tier, + std::string session_id = {}); void RecordFeedback(PolicyFeedbackSample sample); IoPatternSnapshot Snapshot() const; @@ -68,7 +82,27 @@ class IoPatternRuntime final { private: PatternResult AnalyzeWithinBudget(const IoPatternSnapshot& snapshot, - bool& degraded); + bool& degraded); + struct PlannedPolicy { + IoPatternSnapshot snapshot; + PolicyResult result; + bool analysis_degraded{false}; + uint64_t analysis_elapsed_us{0}; + }; + PlannedPolicy BuildPolicy(CacheTier eviction_tier, + uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions, + const std::string& session_id); + void AdmissionWorker(); + ErrorCode ExecuteAdmission(const ObjectRef& object, CacheTier target_tier, + const std::string& session_id); + + struct PendingAdmission { + ObjectRef object; + CacheTier target_tier{CacheTier::kL1Host}; + std::string session_id; + }; Config config_; std::shared_ptr reporter_; @@ -89,6 +123,11 @@ class IoPatternRuntime final { // leave a worker holding a pointer into a destroyed runtime instance. std::shared_ptr> analysis_in_flight_{ std::make_shared>(false)}; + std::mutex admission_mutex_; + std::condition_variable admission_condition_; + std::deque pending_admissions_; + std::thread admission_worker_; + bool admission_stopping_{false}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/sliding_window_analyzer.h b/mooncake-store/include/io_pattern/sliding_window_analyzer.h index 583190d254..dfbd701c6d 100644 --- a/mooncake-store/include/io_pattern/sliding_window_analyzer.h +++ b/mooncake-store/include/io_pattern/sliding_window_analyzer.h @@ -24,8 +24,10 @@ struct WorkloadFeatureStats { class SlidingWindowAnalyzer final : public IoPatternAnalyzer { public: explicit SlidingWindowAnalyzer(uint64_t window_ns = 60'000'000'000ULL, - ThresholdAnalyzerConfig config = {}) + ThresholdAnalyzerConfig config = {}, + size_t max_history_keys = 200'000) : window_ns_(window_ns), + max_history_keys_(max_history_keys), analyzer_(config), kmeans_(KMeansWorkloadAnalyzer::Config{.thresholds = config}) {} @@ -41,8 +43,10 @@ class SlidingWindowAnalyzer final : public IoPatternAnalyzer { void Append(const IoPatternSnapshot& snapshot) const; const uint64_t window_ns_; + const size_t max_history_keys_; mutable std::mutex mutex_; mutable std::deque history_; + mutable size_t history_key_count_{0}; ThresholdAnalyzer analyzer_; KMeansWorkloadAnalyzer kmeans_; }; diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index b73d9ca61c..318a2dd9d5 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include @@ -14,6 +16,17 @@ namespace mooncake { // Forwarded to the HA serve phase via MasterServiceSupervisorConfig. class HttpMetadataServer; +struct IoPatternCfmConfig { + // Empty endpoint means this Master only serves the CFM RPC endpoints. + // Set host:port to report to and poll policies from a central CFM Master. + std::string endpoint; + std::string node_id; + std::string auth_token; + std::string producer_auth_token; + uint32_t timeout_ms{500}; + uint32_t policy_queue_capacity{4096}; +}; + inline std::string ResolveConfiguredHABackendConnstring( std::string_view ha_backend_type, std::string_view ha_backend_connstring, std::string_view etcd_endpoints) { @@ -36,6 +49,7 @@ struct MasterConfig { std::string rpc_interface; int32_t rpc_conn_timeout_seconds; bool rpc_enable_tcp_no_delay; + IoPatternCfmConfig io_pattern_cfm; uint64_t default_kv_lease_ttl; uint64_t default_kv_soft_pin_ttl; @@ -202,6 +216,7 @@ class MasterServiceSupervisorConfig { std::chrono::steady_clock::duration rpc_conn_timeout = std::chrono::seconds( 0); // Client connection timeout. 0 = no timeout (infinite) bool rpc_enable_tcp_no_delay = true; + IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; std::string etcd_endpoints = "0.0.0.0:2379"; @@ -339,6 +354,7 @@ class MasterServiceSupervisorConfig { rpc_conn_timeout = std::chrono::seconds(config.rpc_conn_timeout_seconds); rpc_enable_tcp_no_delay = config.rpc_enable_tcp_no_delay; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; etcd_endpoints = config.etcd_endpoints; ha_backend_connstring = ResolveConfiguredHABackendConnstring( @@ -526,6 +542,7 @@ class WrappedMasterServiceConfig { bool kv_events_emit_legacy_compat = true; bool kv_events_emit_object_key = true; uint32_t kv_events_queue_capacity = 65536; + IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; // OpLog store configuration @@ -616,6 +633,7 @@ class WrappedMasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = ResolveConfiguredHABackendConnstring( ha_backend_type, config.ha_backend_connstring, @@ -733,6 +751,7 @@ class WrappedMasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = ResolveConfiguredHABackendConnstring( ha_backend_type, config.ha_backend_connstring, @@ -841,6 +860,7 @@ class MasterServiceConfigBuilder { std::string cxl_path_ = DEFAULT_CXL_PATH; size_t cxl_size_ = DEFAULT_CXL_SIZE; bool enable_cxl_ = false; + IoPatternCfmConfig io_pattern_cfm_; public: MasterServiceConfigBuilder() = default; @@ -1130,6 +1150,12 @@ class MasterServiceConfigBuilder { return *this; } + MasterServiceConfigBuilder& set_io_pattern_cfm( + IoPatternCfmConfig config) { + io_pattern_cfm_ = std::move(config); + return *this; + } + MasterServiceConfig build() const; }; @@ -1227,6 +1253,7 @@ class MasterServiceConfig { std::string cxl_path = DEFAULT_CXL_PATH; size_t cxl_size = DEFAULT_CXL_SIZE; bool enable_cxl = false; + IoPatternCfmConfig io_pattern_cfm; MasterServiceConfig() = default; // From WrappedMasterServiceConfig @@ -1315,6 +1342,7 @@ class MasterServiceConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; + io_pattern_cfm = config.io_pattern_cfm; } // Static factory method to create a builder @@ -1381,6 +1409,7 @@ inline MasterServiceConfig MasterServiceConfigBuilder::build() const { config.cxl_path = cxl_path_; config.cxl_size = cxl_size_; config.enable_cxl = enable_cxl_; + config.io_pattern_cfm = io_pattern_cfm_; return config; } diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 11ee3fe767..08f223434c 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -47,6 +47,9 @@ namespace mooncake { namespace io_pattern { +class CfmChannel; +class CfmClientImpl; +class CfmService; class IoPatternRuntime; } @@ -334,6 +337,10 @@ class MasterService { bool KvEventsEnabled() const; KvEventPublisher::Stats GetKvEventStats() const; + std::shared_ptr GetCfmService() const { + return io_pattern_cfm_service_; + } + /** * @brief Batch clear KV cache replicas for specified object keys. * @param object_keys Vector of object key strings to clear. @@ -906,7 +913,8 @@ class MasterService { uint64_t evicted_objects{0}; }; TenantQuotaEvictionResult EvictTenantMemoryForQuota( - const TenantId& tenant_id, uint64_t target_bytes); + const TenantId& tenant_id, uint64_t target_bytes, + const std::unordered_set* candidate_keys = nullptr); // Helper to get a snapshot of alive clients (under client_mutex_ shared // lock) @@ -1580,6 +1588,11 @@ class MasterService { const std::chrono::system_clock::time_point& now) -> tl::expected, ErrorCode>; + auto PutEndInternal(const UUID& client_id, const ObjectMeta& object_meta, + const TenantId& tenant_id, ReplicaType replica_type, + uint32_t write_batch_size, bool overwrite) + -> tl::expected; + /** * @brief Helper to discard expired processing keys. */ @@ -2093,7 +2106,14 @@ class MasterService { // The IO-pattern pipeline is deliberately owned by MasterService: the // master has the authoritative replica map and is the only component that // can safely translate a policy plan into promotion/eviction operations. - std::unique_ptr io_pattern_runtime_; + std::shared_ptr io_pattern_runtime_; + std::shared_ptr io_pattern_cfm_service_; + std::shared_ptr io_pattern_cfm_channel_; + std::unique_ptr io_pattern_cfm_client_; + std::atomic io_pattern_cfm_polling_{false}; + std::mutex io_pattern_cfm_poll_mutex_; + std::condition_variable io_pattern_cfm_poll_cv_; + std::thread io_pattern_cfm_poll_thread_; const std::string ha_backend_type_; diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index dc38a091e2..90f479b0ac 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -13,6 +13,7 @@ #include "rpc_types.h" #include "master_config.h" #include "kv_event/kv_event_publisher.h" +#include "io_pattern/cfm_service.h" #include "segment.h" namespace mooncake { @@ -318,8 +319,11 @@ class WrappedMasterService { bool KvEventsEnabled() const; KvEventPublisher::Stats GetKvEventStats() const; + io_pattern::CfmRpcService& CfmRpcEndpoint() { return cfm_rpc_service_; } + private: MasterService master_service_; + io_pattern::CfmRpcService cfm_rpc_service_; }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 095a1b16a4..e7e053104d 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -65,6 +65,7 @@ set(MOONCAKE_STORE_SOURCES io_pattern/reporter.cpp io_pattern/cfm_client_impl.cpp io_pattern/cfm_ingress.cpp + io_pattern/cfm_service.cpp io_pattern/cfm_protocol.cpp io_pattern/resilient_cfm_channel.cpp io_pattern/feedback.cpp diff --git a/mooncake-store/src/io_pattern/cfm_client_impl.cpp b/mooncake-store/src/io_pattern/cfm_client_impl.cpp index 2bd502af2b..88425020fd 100644 --- a/mooncake-store/src/io_pattern/cfm_client_impl.cpp +++ b/mooncake-store/src/io_pattern/cfm_client_impl.cpp @@ -24,8 +24,18 @@ std::optional CfmClientImpl::PollPolicy() { } ErrorCode CfmClientImpl::PollAndDispatchPolicy() { - const auto command = PollPolicy(); - return command ? ReceivePolicy(*command) : ErrorCode::RPC_TIMEOUT; + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + auto result = channel_->PollPolicyResult(); + if (result.status == CfmPollResult::Status::kEmpty) return ErrorCode::OK; + if (result.status == CfmPollResult::Status::kError || !result.command) { + return ErrorCode::RPC_TIMEOUT; + } + const auto execution = ReceivePolicy(*result.command); + if (!channel_->AcknowledgePolicy(result.delivery_id, + execution == ErrorCode::OK)) { + return ErrorCode::RPC_FAIL; + } + return execution; } } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_ingress.cpp b/mooncake-store/src/io_pattern/cfm_ingress.cpp index effabd7b1a..3ce568c6c3 100644 --- a/mooncake-store/src/io_pattern/cfm_ingress.cpp +++ b/mooncake-store/src/io_pattern/cfm_ingress.cpp @@ -1,8 +1,11 @@ #include "io_pattern/cfm_ingress.h" +#include + namespace mooncake::io_pattern { -bool CfmIngress::Handle(std::string_view method, std::string_view payload) { +bool CfmIngress::Handle(std::string_view method, std::string_view payload, + std::string_view source_id) { if (!runtime_ || !codec_) return false; const std::string wire(payload); if (method == "report_snapshot") { @@ -14,14 +17,28 @@ bool CfmIngress::Handle(std::string_view method, std::string_view payload) { if (method == "report_metric_batch") { const auto batch = codec_->DecodeMetricBatch(wire); if (!batch) return false; + const auto received_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); for (const auto& metric : batch->inference) { runtime_->ReportInferenceMetrics(metric); } for (const auto& access : batch->accesses) { - runtime_->RecordAccess(access.object.key, access); + auto normalized = access; + // steady_clock epochs are process-local. CFM observations must be + // rebased to the receiving Store's clock before windowing. + normalized.observed_at_ns = received_at_ns; + runtime_->RecordAccess(normalized.object.key, normalized); } for (const auto& storage : batch->storage) { - runtime_->RecordStorageMetric(storage); + auto normalized = storage; + // The transport identity is authoritative for remote metrics. It + // keeps per-node watermarks distinct even when a producer omitted + // or accidentally reused StorageMetric::source_id. + if (!source_id.empty()) normalized.source_id = source_id; + normalized.observed_at_ns = received_at_ns; + runtime_->RecordStorageMetric(normalized); } return true; } @@ -30,6 +47,10 @@ bool CfmIngress::Handle(std::string_view method, std::string_view payload) { const auto* plan = command ? std::get_if(&*command) : nullptr; return plan && runtime_->ExecuteCommand(*plan) == ErrorCode::OK; } + if (method == "execute_policy") { + const auto command = codec_->DecodePolicy(wire); + return command && runtime_->ExecuteCommand(*command) == ErrorCode::OK; + } return false; } diff --git a/mooncake-store/src/io_pattern/cfm_service.cpp b/mooncake-store/src/io_pattern/cfm_service.cpp new file mode 100644 index 0000000000..f0efe6d048 --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_service.cpp @@ -0,0 +1,284 @@ +#include "io_pattern/cfm_service.h" + +#include +#include +#include + +namespace mooncake::io_pattern { + +CfmService::CfmService(std::shared_ptr runtime, + std::string auth_token, + size_t policy_queue_capacity, + std::string producer_auth_token) + : runtime_(std::move(runtime)), + codec_(std::make_shared()), + ingress_(runtime_, codec_), + auth_token_(std::move(auth_token)), + producer_auth_token_(std::move(producer_auth_token)), + policy_queue_capacity_(policy_queue_capacity), + producer_worker_(&CfmService::PolicyProducerWorker, this) {} + +CfmService::~CfmService() { + { + std::lock_guard lock(producer_mutex_); + producer_stopping_ = true; + pending_metric_batches_.clear(); + } + producer_cv_.notify_all(); + if (producer_worker_.joinable()) producer_worker_.join(); +} + +bool CfmService::Authenticate(std::string_view token) const { + return AuthenticateNode(token) || AuthenticateProducer(token); +} + +bool CfmService::AuthenticateNode(std::string_view token) const { + return !auth_token_.empty() && token == auth_token_; +} + +bool CfmService::AuthenticateProducer(std::string_view token) const { + return !producer_auth_token_.empty() && producer_auth_token_ != auth_token_ && + token == producer_auth_token_; +} + +bool CfmService::Send(std::string_view node_id, std::string_view method, + std::string_view payload, std::string_view token) { + const bool executes_policy = + method == "execute_policy" || method == "execute_prefetch"; + if (node_id.empty() || + (executes_policy ? !AuthenticateProducer(token) + : !AuthenticateNode(token))) { + return false; + } + std::optional metric_batch; + if (method == "report_metric_batch") { + metric_batch = codec_->DecodeMetricBatch(std::string(payload)); + if (!metric_batch) return false; + } + if (!ingress_.Handle(method, payload, node_id)) return false; + if (metric_batch) { + SchedulePolicyProduction(std::string(node_id), + std::move(*metric_batch)); + } + return true; +} + +std::optional> CfmService::PollPolicy( + std::string_view node_id, std::string_view token) { + if (!AuthenticateNode(token) || node_id.empty()) return std::nullopt; + std::lock_guard lock(mutex_); + auto it = policy_queues_.find(std::string(node_id)); + if (it == policy_queues_.end() || it->second.empty()) return std::nullopt; + return it->second.front(); +} + +bool CfmService::AcknowledgePolicy(std::string_view node_id, + uint64_t delivery_id, bool success, + std::string_view token) { + if (!AuthenticateNode(token) || node_id.empty() || delivery_id == 0) { + return false; + } + std::lock_guard lock(mutex_); + auto it = policy_queues_.find(std::string(node_id)); + if (it == policy_queues_.end() || it->second.empty() || + it->second.front().first != delivery_id) { + return false; + } + if (!success) { + if (it->second.size() > 1) { + auto failed = std::move(it->second.front()); + it->second.pop_front(); + it->second.push_back(std::move(failed)); + } + return true; + } + it->second.pop_front(); + --total_queued_policies_; + if (it->second.empty()) policy_queues_.erase(it); + return true; +} + +bool CfmService::EnqueuePolicy(std::string node_id, std::string payload, + std::string_view token) { + if (!AuthenticateProducer(token) || node_id.empty() || payload.empty() || + !codec_->DecodePolicy(payload)) { + return false; + } + return EnqueueValidated(std::move(node_id), std::move(payload)); +} + +bool CfmService::EnqueueValidated(std::string node_id, std::string payload) { + std::lock_guard lock(mutex_); + auto& queue = policy_queues_[node_id]; + if (std::any_of(queue.begin(), queue.end(), [&](const auto& queued) { + return queued.second == payload; + })) { + return true; + } + if (policy_queue_capacity_ == 0 || + total_queued_policies_ >= policy_queue_capacity_ || + queue.size() >= policy_queue_capacity_) { + if (queue.empty()) policy_queues_.erase(node_id); + return false; + } + if (next_delivery_id_ == 0) next_delivery_id_ = 1; + const uint64_t delivery_id = next_delivery_id_++; + queue.emplace_back(delivery_id, std::move(payload)); + ++total_queued_policies_; + return true; +} + +void CfmService::SchedulePolicyProduction(std::string node_id, + MetricBatch batch) { + { + std::lock_guard lock(producer_mutex_); + if (producer_stopping_ || policy_queue_capacity_ == 0 || + pending_metric_batches_.size() >= policy_queue_capacity_) { + return; + } + pending_metric_batches_.emplace_back(std::move(node_id), + std::move(batch)); + } + producer_cv_.notify_one(); +} + +void CfmService::PolicyProducerWorker() { + while (true) { + std::pair pending; + { + std::unique_lock lock(producer_mutex_); + producer_cv_.wait(lock, [this] { + return producer_stopping_ || !pending_metric_batches_.empty(); + }); + if (producer_stopping_) return; + pending = std::move(pending_metric_batches_.front()); + pending_metric_batches_.pop_front(); + } + try { + ProducePolicies(pending.first, pending.second); + } catch (...) { + // Policy production is best effort and must never terminate the + // RPC service. The next metric batch will trigger a fresh plan. + } + } +} + +void CfmService::ProducePolicies(std::string_view node_id, + const MetricBatch& batch) { + if (!runtime_ || node_id.empty()) return; + + std::unordered_map match_lengths; + std::string session_id; + for (const auto& metric : batch.inference) { + match_lengths[metric.object] = metric.match_length; + if (session_id.empty()) session_id = metric.session_id; + } + TraceHistory trace; + trace.events.reserve(batch.accesses.size()); + for (const auto& access : batch.accesses) { + const auto match = match_lengths.find(access.object); + trace.events.push_back( + {.object = access.object, + .observed_at_ns = access.observed_at_ns, + .match_length = match == match_lengths.end() ? 0U + : match->second, + .is_hit = access.is_hit}); + } + + bool produced_prefetch = false; + for (const auto& storage : batch.storage) { + if (static_cast(storage.tier) > + static_cast(CacheTier::kL3NofSsd)) { + continue; + } + if (!std::isfinite(storage.memory_used_ratio) || + storage.memory_used_ratio < 0.90F) { + continue; + } + const float used_ratio = + std::clamp(storage.memory_used_ratio, 0.0F, 1.0F); + uint64_t target_bytes = 0; + if (storage.capacity_bytes != 0) { + const uint64_t low_watermark = + storage.capacity_bytes - storage.capacity_bytes / 5; + if (storage.used_bytes > low_watermark) { + target_bytes = storage.used_bytes - low_watermark; + } + } + if (target_bytes == 0) { + uint64_t tier_bytes = 0; + for (const auto& key : runtime_->Snapshot().keys) { + if ((key.replica_tiers & CacheTierBit(storage.tier)) == 0) { + continue; + } + tier_bytes = + key.block_size > + std::numeric_limits::max() - tier_bytes + ? std::numeric_limits::max() + : tier_bytes + key.block_size; + } + const auto excess_ratio = std::max( + 0.0F, used_ratio - 0.80F); + target_bytes = static_cast( + static_cast(tier_bytes) * excess_ratio / + std::max(0.01F, used_ratio)); + } + auto result = runtime_->Plan(storage.tier, target_bytes, trace, {}, + session_id); + if (result.degraded) continue; + if (!result.eviction.candidates.empty()) { + EnqueueValidated(std::string(node_id), + codec_->EncodePolicy(result.eviction)); + } + if (!produced_prefetch && !result.prefetch.candidates.empty()) { + EnqueueValidated(std::string(node_id), + codec_->EncodePolicy(result.prefetch)); + produced_prefetch = true; + } + } + + if (!produced_prefetch && !trace.events.empty()) { + auto result = runtime_->Plan(CacheTier::kL1Host, 0, trace, {}, + session_id); + if (!result.degraded && !result.prefetch.candidates.empty()) { + EnqueueValidated(std::string(node_id), + codec_->EncodePolicy(result.prefetch)); + } + } +} + +bool CfmRpcService::Authenticate(const std::string& auth_token) { + return service_ && service_->Authenticate(auth_token); +} + +bool CfmRpcService::Send(const std::string& node_id, const std::string& method, + const std::string& payload, + const std::string& auth_token) { + return service_ && service_->Send(node_id, method, payload, auth_token); +} + +std::pair>> +CfmRpcService::Receive( + const std::string& method, const std::string& node_id, + const std::string& auth_token) { + if (!service_ || method != "poll_policy" || + !service_->AuthenticateNode(auth_token) || node_id.empty()) { + return {false, std::nullopt}; + } + return {true, service_->PollPolicy(node_id, auth_token)}; +} + +bool CfmRpcService::Acknowledge(const std::string& node_id, + uint64_t delivery_id, bool success, + const std::string& auth_token) { + return service_ && service_->AcknowledgePolicy( + node_id, delivery_id, success, auth_token); +} + +bool CfmRpcService::EnqueuePolicy(const std::string& node_id, + const std::string& payload, + const std::string& auth_token) { + return service_ && service_->EnqueuePolicy(node_id, payload, auth_token); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/collector_impl.cpp b/mooncake-store/src/io_pattern/collector_impl.cpp index e31b06c223..ff03f54ade 100644 --- a/mooncake-store/src/io_pattern/collector_impl.cpp +++ b/mooncake-store/src/io_pattern/collector_impl.cpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace mooncake::io_pattern { @@ -13,6 +14,43 @@ uint64_t NowNs() { } } +void IoPatternCollectorImpl::ApplyAccessWindow(const ObjectRef& object, + uint64_t now_ns, + KeyMetrics& metrics) const { + metrics.access_count_window = 0; + metrics.write_frequency = 0; + metrics.overwrite_ratio = 0.0F; + metrics.write_batch_size = 0; + metrics.write_burst = false; + const auto it = access_windows_.find(object); + if (it == access_windows_.end()) return; + + const uint64_t cutoff = config_.access_window_ns != 0 && + now_ns > config_.access_window_ns + ? now_ns - config_.access_window_ns + : 0; + uint64_t writes = 0; + uint64_t overwrites = 0; + for (const auto& bucket : it->second) { + if (config_.access_window_ns != 0 && bucket.observed_at_ns < cutoff) { + continue; + } + metrics.access_count_window += bucket.access_count; + writes += bucket.write_count; + overwrites += bucket.overwrite_count; + metrics.write_batch_size = + std::max(metrics.write_batch_size, bucket.max_write_batch_size); + } + metrics.write_frequency = + static_cast(std::min( + writes, std::numeric_limits::max())); + if (writes != 0) { + metrics.overwrite_ratio = static_cast(overwrites) / + static_cast(writes); + } + metrics.write_burst = metrics.write_batch_size >= 16; +} + void IoPatternCollectorImpl::ReportInferenceMetrics( const InferenceMetrics& metrics) { std::lock_guard lock(mutex_); @@ -66,32 +104,70 @@ void IoPatternCollectorImpl::RecordAccess(const std::string& key, if (!key_metrics_.contains(object)) ++tenant_key_counts_[object.tenant_id]; auto& value = key_metrics_[object]; value.object = object; - ++value.access_count_window; + const uint64_t observed_at_ns = + record.observed_at_ns == 0 + ? (config_.now_ns ? config_.now_ns() : NowNs()) + : record.observed_at_ns; + auto& window = access_windows_[object]; + const uint64_t bucket_ns = std::max(1, config_.access_bucket_ns); + const uint64_t bucket_timestamp = + observed_at_ns - (observed_at_ns % bucket_ns); + const uint64_t reference_ns = + window.empty() ? observed_at_ns + : std::max(observed_at_ns, + window.back().observed_at_ns); + AccessWindowBucket bucket{.observed_at_ns = bucket_timestamp, + .access_count = 1}; + if (record.operation == IoOperation::kPut) { + bucket.write_count = 1; + bucket.overwrite_count = record.overwrite ? 1 : 0; + bucket.max_write_batch_size = record.write_batch_size; + } + const auto position = std::lower_bound( + window.begin(), window.end(), bucket_timestamp, + [](const AccessWindowBucket& existing, uint64_t timestamp) { + return existing.observed_at_ns < timestamp; + }); + if (position != window.end() && + position->observed_at_ns == bucket_timestamp) { + position->access_count += bucket.access_count; + position->write_count += bucket.write_count; + position->overwrite_count += bucket.overwrite_count; + position->max_write_batch_size = std::max( + position->max_write_batch_size, bucket.max_write_batch_size); + } else { + window.insert(position, bucket); + } + const uint64_t cutoff = config_.access_window_ns != 0 && + reference_ns > config_.access_window_ns + ? reference_ns - config_.access_window_ns + : 0; + while (!window.empty() && config_.access_window_ns != 0 && + window.front().observed_at_ns < cutoff) { + window.pop_front(); + } + while (config_.max_access_buckets_per_key != 0 && + window.size() > config_.max_access_buckets_per_key) { + window.pop_front(); + } value.last_access_time_ns = - std::max(value.last_access_time_ns, record.observed_at_ns); + std::max(value.last_access_time_ns, observed_at_ns); value.block_size = std::max(value.block_size, record.block_size); value.replica_tiers |= CacheTierBit(record.tier); value.active = value.active || record.is_hit; if (record.operation == IoOperation::kPut) { - ++write_counts_[object]; - if (record.overwrite) ++overwrite_counts_[object]; - value.write_frequency = - static_cast(std::min(write_counts_[object], - UINT32_MAX)); - value.write_batch_size = - std::max(value.write_batch_size, record.write_batch_size); value.write_object_size = std::max(value.write_object_size, record.block_size); - value.overwrite_ratio = - static_cast(overwrite_counts_[object]) / - static_cast(write_counts_[object]); - value.write_burst = record.write_batch_size >= 16; } + ApplyAccessWindow(object, observed_at_ns, value); } void IoPatternCollectorImpl::RecordStorageMetric(const StorageMetric& metric) { std::lock_guard lock(mutex_); - if (reporter_ && !reporter_->EnqueueStorage(metric)) ++dropped_; + if (reporter_) { + reporter_->UpdateLoad(metric.memory_used_ratio, metric.rpc_latency_us); + if (!reporter_->EnqueueStorage(metric)) ++dropped_; + } StorageMetricKey key{metric.source_id, metric.tier}; auto it = storage_metrics_.find(key); if (it == storage_metrics_.end() || @@ -102,6 +178,7 @@ void IoPatternCollectorImpl::RecordStorageMetric(const StorageMetric& metric) { void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { std::lock_guard lock(mutex_); + const uint64_t received_at_ns = config_.now_ns ? config_.now_ns() : NowNs(); for (const auto& metrics : snapshot.keys) { if (!key_metrics_.contains(metrics.object) && config_.max_total_keys != 0 && @@ -121,6 +198,15 @@ void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { ++tenant_key_counts_[metrics.object.tenant_id]; } key_metrics_[metrics.object] = metrics; + auto& window = access_windows_[metrics.object]; + window.clear(); + window.push_back({.observed_at_ns = received_at_ns, + .access_count = metrics.access_count_window, + .write_count = metrics.write_frequency, + .overwrite_count = static_cast( + metrics.overwrite_ratio * + static_cast(metrics.write_frequency)), + .max_write_batch_size = metrics.write_batch_size}); } for (const auto& metric : snapshot.storage) { StorageMetricKey key{metric.source_id, metric.tier}; @@ -135,11 +221,11 @@ void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { IoPatternSnapshot IoPatternCollectorImpl::GetSnapshot() const { std::lock_guard lock(mutex_); IoPatternSnapshot snapshot; - snapshot.generated_at_ns = NowNs(); + snapshot.generated_at_ns = config_.now_ns ? config_.now_ns() : NowNs(); snapshot.keys.reserve(key_metrics_.size()); for (const auto& [object, metrics] : key_metrics_) { - (void)object; auto copy = metrics; + ApplyAccessWindow(object, snapshot.generated_at_ns, copy); if (copy.last_access_time_ns != 0 && snapshot.generated_at_ns > copy.last_access_time_ns) { copy.idle_time_us = diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index 98e00f308c..d11a542bfc 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -1,6 +1,7 @@ #include "io_pattern/policy_strategies.h" #include +#include #include namespace mooncake::io_pattern { @@ -122,11 +123,11 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, } uint64_t selected_bytes = 0; auto end = plan.candidates.begin(); - while (end != plan.candidates.end()) { - if (end->bytes > target_bytes - selected_bytes) { - break; - } - selected_bytes += end->bytes; + while (end != plan.candidates.end() && selected_bytes < target_bytes) { + selected_bytes = + end->bytes > std::numeric_limits::max() - selected_bytes + ? std::numeric_limits::max() + : selected_bytes + end->bytes; ++end; } plan.candidates.erase(end, plan.candidates.end()); @@ -156,10 +157,14 @@ AdmissionResult PrefixMatchAdmissionOps::Evaluate( result.decision = key->access_count_window >= config_.frequency_threshold ? AdmissionDecision::kAdmit : AdmissionDecision::kRejectFrequency; + const bool target_over_watermark = std::any_of( + context.snapshot.storage.begin(), context.snapshot.storage.end(), + [&](const StorageMetric& metric) { + return metric.tier == target_tier && + metric.memory_used_ratio >= config_.max_memory_used_ratio; + }); if (result.decision == AdmissionDecision::kAdmit && - !context.snapshot.storage.empty() && - context.snapshot.storage.front().memory_used_ratio >= - config_.max_memory_used_ratio) { + target_over_watermark) { result.decision = AdmissionDecision::kRejectWatermark; } result.confidence = diff --git a/mooncake-store/src/io_pattern/reporter.cpp b/mooncake-store/src/io_pattern/reporter.cpp index c8bf7f4fe8..a9428a45a8 100644 --- a/mooncake-store/src/io_pattern/reporter.cpp +++ b/mooncake-store/src/io_pattern/reporter.cpp @@ -19,13 +19,7 @@ void IoPatternReporter::Start() { worker_ = std::thread([this] { std::unique_lock lock(mutex_); while (running_) { - const size_t size = batch_.inference.size() + batch_.accesses.size() + - batch_.storage.size(); - const auto interval = - (capacity_ == 0 || size * 2 >= capacity_) - ? std::chrono::milliseconds(100) - : (size == 0 ? std::chrono::milliseconds(1000) - : std::chrono::milliseconds(500)); + const auto interval = RecommendedFlushIntervalLocked(); condition_.wait_for(lock, interval, [this] { return !running_; }); if (!running_) break; lock.unlock(); @@ -131,13 +125,25 @@ uint64_t IoPatternReporter::reported() const { std::chrono::milliseconds IoPatternReporter::RecommendedFlushInterval() const { std::lock_guard lock(mutex_); - const size_t size = batch_.inference.size() + batch_.accesses.size() + - batch_.storage.size(); - if (capacity_ == 0 || size * 2 >= capacity_) { - return std::chrono::milliseconds(100); + return RecommendedFlushIntervalLocked(); +} + +void IoPatternReporter::UpdateLoad(float memory_used_ratio, + uint64_t rpc_latency_us) { + std::lock_guard lock(mutex_); + memory_used_ratio_ = memory_used_ratio; + rpc_latency_us_ = rpc_latency_us; + condition_.notify_one(); +} + +std::chrono::milliseconds +IoPatternReporter::RecommendedFlushIntervalLocked() const { + if (memory_used_ratio_ >= 0.95F || rpc_latency_us_ > 100'000) { + return std::chrono::milliseconds(1000); } - if (size == 0) return std::chrono::milliseconds(1000); - return std::chrono::milliseconds(500); + if (memory_used_ratio_ >= 0.80F) return std::chrono::milliseconds(500); + if (memory_used_ratio_ >= 0.50F) return std::chrono::milliseconds(200); + return std::chrono::milliseconds(100); } } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp index 4786cdad53..013f7e725d 100644 --- a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp +++ b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp @@ -22,20 +22,31 @@ bool ResilientCfmChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { return Retry([&] { return delegate_->SendSnapshot(snapshot); }); } -std::optional ResilientCfmChannel::PollPolicy() { +CfmPollResult ResilientCfmChannel::PollPolicyResult() { if (!delegate_) { RecordFailure(); - return std::nullopt; + return CfmPollResult::Error(); } for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { - auto result = delegate_->PollPolicy(); - if (result.has_value()) { + auto result = delegate_->PollPolicyResult(); + if (result.status == CfmPollResult::Status::kCommand) { + RecordSuccess(); + return result; + } + if (result.status == CfmPollResult::Status::kEmpty) { RecordSuccess(); return result; } } RecordFailure(); - return std::nullopt; + return CfmPollResult::Error(); +} + +bool ResilientCfmChannel::AcknowledgePolicy(uint64_t delivery_id, + bool success) { + return Retry([&] { + return delegate_->AcknowledgePolicy(delivery_id, success); + }); } ErrorCode ResilientCfmChannel::ExecutePrefetch(const PrefetchPlan& plan) { diff --git a/mooncake-store/src/io_pattern/rpc_transport.cpp b/mooncake-store/src/io_pattern/rpc_transport.cpp index 429d894bae..8f9e992890 100644 --- a/mooncake-store/src/io_pattern/rpc_transport.cpp +++ b/mooncake-store/src/io_pattern/rpc_transport.cpp @@ -1,7 +1,155 @@ #include "io_pattern/rpc_transport.h" +#include +#include +#include + +#include + +#include "io_pattern/cfm_service.h" +#include "store_rpc_client_io_context.h" + namespace mooncake::io_pattern { +class CoroRpcCfmTransport::Impl { + public: + Impl(std::string endpoint, std::string node_id, + std::chrono::milliseconds default_timeout) + : endpoint_(std::move(endpoint)), + node_id_(std::move(node_id)), + default_timeout_(default_timeout) {} + + template + std::optional Invoke(std::chrono::milliseconds timeout, + Args&&... args) { + auto pool = GetPool(timeout.count() > 0 ? timeout : default_timeout_); + return async_simple::coro::syncAwait( + [&]() -> async_simple::coro::Lazy> { + auto request = co_await pool->send_request( + [&](coro_io::client_reuse_hint, + coro_rpc::coro_rpc_client& client) { + return client.send_request( + std::forward(args)...); + }); + if (!request) co_return std::nullopt; + auto response = co_await std::move(request.value()); + if (!response) co_return std::nullopt; + co_return response->result(); + }()); + } + + std::shared_ptr> GetPool( + std::chrono::milliseconds timeout) { + std::lock_guard lock(mutex_); + const auto key = timeout.count(); + const auto existing = pools_.find(key); + if (existing != pools_.end()) return existing->second; + coro_io::client_pool::pool_config config; + config.client_config.request_timeout_duration = timeout; + config.host_alive_detect_duration = std::chrono::seconds(0); + auto pool = coro_io::client_pool::create( + endpoint_, config, GetStoreRpcClientIoContextPool()); + pools_.emplace(key, pool); + return pool; + } + + std::string endpoint_; + std::string node_id_; + std::chrono::milliseconds default_timeout_; + std::mutex mutex_; + std::string auth_token_; + std::unordered_map< + int64_t, + std::shared_ptr>> + pools_; +}; + +CoroRpcCfmTransport::CoroRpcCfmTransport( + std::string endpoint, std::string node_id, + std::chrono::milliseconds default_timeout) + : impl_(std::make_unique(std::move(endpoint), std::move(node_id), + default_timeout)) {} + +CoroRpcCfmTransport::~CoroRpcCfmTransport() = default; + +bool CoroRpcCfmTransport::Authenticate(std::string_view token) { + if (!impl_ || token.empty()) return false; + const std::string wire_token(token); + const auto result = impl_->Invoke<&CfmRpcService::Authenticate, bool>( + impl_->default_timeout_, wire_token); + if (!result || !*result) return false; + std::lock_guard lock(impl_->mutex_); + impl_->auth_token_ = wire_token; + return true; +} + +bool CoroRpcCfmTransport::Send(std::string_view method, + std::string_view payload, + std::chrono::milliseconds timeout) { + if (!impl_) return false; + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return false; + const auto result = impl_->Invoke<&CfmRpcService::Send, bool>( + timeout, impl_->node_id_, std::string(method), std::string(payload), + auth_token); + return result && *result; +} + +CfmReceiveResult CoroRpcCfmTransport::Receive( + std::string_view method, std::chrono::milliseconds timeout) { + if (!impl_) return CfmReceiveResult::Error(); + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return CfmReceiveResult::Error(); + const auto result = + impl_->Invoke<&CfmRpcService::Receive, + std::pair< + bool, + std::optional>>>( + timeout, std::string(method), impl_->node_id_, auth_token); + if (!result || !result->first) return CfmReceiveResult::Error(); + if (!result->second) return CfmReceiveResult::Empty(); + return CfmReceiveResult::Payload(std::move(result->second->second), + result->second->first); +} + +bool CoroRpcCfmTransport::Acknowledge( + uint64_t delivery_id, bool success, std::chrono::milliseconds timeout) { + if (!impl_ || delivery_id == 0) return false; + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return false; + const auto result = impl_->Invoke<&CfmRpcService::Acknowledge, bool>( + timeout, impl_->node_id_, delivery_id, success, auth_token); + return result && *result; +} + +bool CoroRpcCfmTransport::EnqueuePolicy( + std::string_view node_id, std::string_view payload, + std::chrono::milliseconds timeout) { + if (!impl_) return false; + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return false; + const auto result = + impl_->Invoke<&CfmRpcService::EnqueuePolicy, bool>( + timeout, std::string(node_id), std::string(payload), auth_token); + return result && *result; +} + bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { std::lock_guard lock(mutex_); authenticated_ = token == auth_token_; @@ -11,20 +159,23 @@ bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { bool InProcessCfmRpcTransport::Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) { - std::lock_guard lock(mutex_); - if (!authenticated_) return false; - return !send_handler_ || send_handler_(method, payload); + SendHandler handler; + { + std::lock_guard lock(mutex_); + if (!authenticated_) return false; + handler = send_handler_; + } + return !handler || handler(method, payload); } -std::optional InProcessCfmRpcTransport::Receive( +CfmReceiveResult InProcessCfmRpcTransport::Receive( std::string_view method, std::chrono::milliseconds) { std::lock_guard lock(mutex_); - if (!authenticated_ || method != "poll_policy" || policies_.empty()) { - return std::nullopt; - } + if (!authenticated_ || method != "poll_policy") return CfmReceiveResult::Error(); + if (policies_.empty()) return CfmReceiveResult::Empty(); auto payload = std::move(policies_.front()); policies_.pop(); - return payload; + return CfmReceiveResult::Payload(std::move(payload)); } void InProcessCfmRpcTransport::EnqueuePolicy(std::string payload) { @@ -50,10 +201,28 @@ bool CfmRpcChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { config_.timeout); } -std::optional CfmRpcChannel::PollPolicy() { - if (!transport_ || !codec_ || !EnsureAuthenticated()) return std::nullopt; - const auto payload = transport_->Receive("poll_policy", config_.timeout); - return payload ? codec_->DecodePolicy(*payload) : std::nullopt; +CfmPollResult CfmRpcChannel::PollPolicyResult() { + if (!transport_ || !codec_ || !EnsureAuthenticated()) { + return CfmPollResult::Error(); + } + auto received = transport_->Receive("poll_policy", config_.timeout); + if (received.status == CfmReceiveResult::Status::kEmpty) { + return CfmPollResult::Empty(); + } + if (received.status == CfmReceiveResult::Status::kError) { + return CfmPollResult::Error(); + } + auto command = codec_->DecodePolicy(received.payload); + if (!command) { + transport_->Acknowledge(received.delivery_id, false, config_.timeout); + return CfmPollResult::Error(); + } + return CfmPollResult::Command(std::move(*command), received.delivery_id); +} + +bool CfmRpcChannel::AcknowledgePolicy(uint64_t delivery_id, bool success) { + return transport_ && delivery_id != 0 && EnsureAuthenticated() && + transport_->Acknowledge(delivery_id, success, config_.timeout); } ErrorCode CfmRpcChannel::ExecutePrefetch(const PrefetchPlan& plan) { @@ -87,14 +256,26 @@ bool CfmChannelPool::SendSnapshot(const IoPatternSnapshot& snapshot) { return false; } -std::optional CfmChannelPool::PollPolicy() { +CfmPollResult CfmChannelPool::PollPolicyResult() { + bool saw_empty = false; for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { auto channel = Next(); if (!channel) continue; - auto command = channel->PollPolicy(); - if (command) return command; + auto result = channel->PollPolicyResult(); + if (result.status == CfmPollResult::Status::kCommand) return result; + saw_empty = saw_empty || result.status == CfmPollResult::Status::kEmpty; + } + return saw_empty ? CfmPollResult::Empty() : CfmPollResult::Error(); +} + +bool CfmChannelPool::AcknowledgePolicy(uint64_t delivery_id, bool success) { + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (channel && channel->AcknowledgePolicy(delivery_id, success)) { + return true; + } } - return std::nullopt; + return false; } ErrorCode CfmChannelPool::ExecutePrefetch(const PrefetchPlan& plan) { diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index f2a41f2cae..1e4bc31d02 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -38,9 +38,17 @@ IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) std::make_shared(std::move(legacy_strategy)), nullptr, std::make_shared()); policy_ = std::make_shared(workload_policy_, fallback); + admission_worker_ = std::thread(&IoPatternRuntime::AdmissionWorker, this); } IoPatternRuntime::~IoPatternRuntime() { + { + std::lock_guard lock(admission_mutex_); + admission_stopping_ = true; + pending_admissions_.clear(); + } + admission_condition_.notify_all(); + if (admission_worker_.joinable()) admission_worker_.join(); if (reporter_) reporter_->Stop(); } @@ -160,32 +168,18 @@ PatternResult IoPatternRuntime::AnalyzeWithinBudget( PolicyExecutionStatus IoPatternRuntime::Execute( CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, const std::vector& admissions, const std::string& session_id) { - const auto snapshot = collector_->GetSnapshot(); - const auto start = std::chrono::steady_clock::now(); - bool analysis_degraded = false; - const auto analysis = AnalyzeWithinBudget(snapshot, analysis_degraded); - const auto elapsed = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start) - .count(); - observability_.RecordAnalyzeLatency(elapsed); - - workload_policy_->SetWorkloadType(analysis.workload_type); - workload_policy_->SetSessionWorkloads(analysis.sessions); - workload_policy_->AdvanceTransitionWindow(); - const PolicyResult result = policy_->ExecutePolicy( - PolicyContext{.snapshot = snapshot, .analysis = analysis, - .session_id = session_id}, eviction_tier, - eviction_bytes, trace, admissions); + auto planned = BuildPolicy(eviction_tier, eviction_bytes, trace, admissions, + session_id); + const auto& snapshot = planned.snapshot; + const auto& result = planned.result; auto status = executor_.Execute(result); - status.degraded = status.degraded || result.degraded || collector_->degraded() || - analysis_degraded || - elapsed > static_cast(config_.analysis_timeout_us); - observability_.RecordPolicyDecision(!result.eviction.candidates.empty() || - !result.prefetch.candidates.empty()); + status.degraded = status.degraded || result.degraded; const bool failed = status.eviction != ErrorCode::OK || status.prefetch != ErrorCode::OK || status.degraded; - if (failed) policy_->RecordFailure(); - else policy_->RecordSuccess(); + if (failed) + policy_->RecordFailure(); + else + policy_->RecordSuccess(); if (status.degraded || policy_->degraded()) observability_.RecordDegrade(); status.degraded = status.degraded || policy_->degraded(); @@ -199,7 +193,8 @@ PolicyExecutionStatus IoPatternRuntime::Execute( pending_prefetches_.insert(candidate.object); } } - if (!result.prefetch.candidates.empty() && status.prefetch != ErrorCode::OK) { + if (!result.prefetch.candidates.empty() && + status.prefetch != ErrorCode::OK) { feedback.prefetch_accuracy = 0.0F; has_feedback = true; } @@ -214,6 +209,45 @@ PolicyExecutionStatus IoPatternRuntime::Execute( return status; } +PolicyResult IoPatternRuntime::Plan( + CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, + const std::vector& admissions, const std::string& session_id) { + return BuildPolicy(eviction_tier, eviction_bytes, trace, admissions, + session_id) + .result; +} + +IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( + CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, + const std::vector& admissions, const std::string& session_id) { + PlannedPolicy planned; + planned.snapshot = collector_->GetSnapshot(); + const auto start = std::chrono::steady_clock::now(); + const auto analysis = + AnalyzeWithinBudget(planned.snapshot, planned.analysis_degraded); + planned.analysis_elapsed_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + observability_.RecordAnalyzeLatency(planned.analysis_elapsed_us); + + workload_policy_->SetWorkloadType(analysis.workload_type); + workload_policy_->SetSessionWorkloads(analysis.sessions); + workload_policy_->AdvanceTransitionWindow(); + planned.result = policy_->ExecutePolicy( + PolicyContext{.snapshot = planned.snapshot, .analysis = analysis, + .session_id = session_id}, eviction_tier, + eviction_bytes, trace, admissions); + planned.result.degraded = + planned.result.degraded || collector_->degraded() || + planned.analysis_degraded || + planned.analysis_elapsed_us > config_.analysis_timeout_us; + observability_.RecordPolicyDecision( + !planned.result.eviction.candidates.empty() || + !planned.result.prefetch.candidates.empty()); + return planned; +} + ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { PolicyResult result; if (const auto* eviction = std::get_if(&command)) { @@ -237,6 +271,70 @@ ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { return status.admissions.empty() ? ErrorCode::OK : status.admissions.front(); } +bool IoPatternRuntime::ScheduleAdmission(ObjectRef object, CacheTier target_tier, + std::string session_id) { + { + std::lock_guard lock(admission_mutex_); + if (admission_stopping_ || + (config_.max_pending_admissions != 0 && + pending_admissions_.size() >= config_.max_pending_admissions)) { + return false; + } + pending_admissions_.push_back( + {.object = std::move(object), + .target_tier = target_tier, + .session_id = std::move(session_id)}); + } + admission_condition_.notify_one(); + return true; +} + +void IoPatternRuntime::AdmissionWorker() { + while (true) { + PendingAdmission pending; + { + std::unique_lock lock(admission_mutex_); + admission_condition_.wait(lock, [this] { + return admission_stopping_ || !pending_admissions_.empty(); + }); + if (admission_stopping_) return; + pending = std::move(pending_admissions_.front()); + pending_admissions_.pop_front(); + } + try { + ExecuteAdmission(pending.object, pending.target_tier, + pending.session_id); + } catch (...) { + policy_->RecordFailure(); + observability_.RecordDegrade(); + } + } +} + +ErrorCode IoPatternRuntime::ExecuteAdmission(const ObjectRef& object, + CacheTier target_tier, + const std::string& session_id) { + const auto snapshot = collector_->GetSnapshot(); + bool analysis_degraded = false; + const auto analysis = AnalyzeWithinBudget(snapshot, analysis_degraded); + workload_policy_->SetWorkloadType(analysis.workload_type); + workload_policy_->SetSessionWorkloads(analysis.sessions); + const auto admission = policy_->DecideAdmission( + object, target_tier, + PolicyContext{.snapshot = snapshot, + .analysis = analysis, + .session_id = session_id}); + PolicyResult result; + result.admissions.push_back(admission); + auto status = executor_.Execute(result); + if (analysis_degraded) status.degraded = true; + const auto code = status.admissions.empty() ? ErrorCode::OK + : status.admissions.front(); + if (code != ErrorCode::OK || status.degraded) + observability_.RecordDegrade(); + return code; +} + void IoPatternRuntime::RecordFeedback(PolicyFeedbackSample sample) { feedback_.Record(sample); auto config = workload_policy_->CurrentEvictionConfig(); diff --git a/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp index ed992fc45e..2020f8d021 100644 --- a/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp +++ b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp @@ -1,6 +1,7 @@ #include "io_pattern/sliding_window_analyzer.h" #include +#include #include namespace mooncake::io_pattern { @@ -11,19 +12,60 @@ T Percentile(std::vector values, size_t rank) { std::sort(values.begin(), values.end()); return values[std::min(rank, values.size() - 1)]; } + +std::vector LatestKeys( + const std::deque& history) { + std::unordered_map latest; + for (const auto& snapshot : history) { + for (const auto& key : snapshot.keys) latest[key.object] = key; + } + std::vector keys; + keys.reserve(latest.size()); + for (auto& [object, key] : latest) { + (void)object; + keys.push_back(std::move(key)); + } + std::sort(keys.begin(), keys.end(), [](const KeyMetrics& left, + const KeyMetrics& right) { + if (left.object.tenant_id != right.object.tenant_id) { + return left.object.tenant_id < right.object.tenant_id; + } + return left.object.key < right.object.key; + }); + return keys; +} } void SlidingWindowAnalyzer::Append(const IoPatternSnapshot& snapshot) const { std::lock_guard lock(mutex_); + IoPatternSnapshot bounded = snapshot; + if (max_history_keys_ != 0 && + bounded.keys.size() > max_history_keys_) { + std::sort(bounded.keys.begin(), bounded.keys.end(), + [](const KeyMetrics& left, const KeyMetrics& right) { + if (left.object.tenant_id != right.object.tenant_id) { + return left.object.tenant_id < right.object.tenant_id; + } + return left.object.key < right.object.key; + }); + bounded.keys.resize(max_history_keys_); + } if (history_.empty() || - history_.back().generated_at_ns != snapshot.generated_at_ns) { - history_.push_back(snapshot); + history_.back().generated_at_ns != bounded.generated_at_ns) { + history_key_count_ += bounded.keys.size(); + history_.push_back(std::move(bounded)); } const uint64_t cutoff = snapshot.generated_at_ns > window_ns_ ? snapshot.generated_at_ns - window_ns_ : 0; - while (!history_.empty() && history_.front().generated_at_ns < cutoff) + while (!history_.empty() && history_.front().generated_at_ns < cutoff) { + history_key_count_ -= history_.front().keys.size(); history_.pop_front(); + } + while (max_history_keys_ != 0 && history_key_count_ > max_history_keys_) { + history_key_count_ -= history_.front().keys.size(); + history_.pop_front(); + } } IoPatternSnapshot SlidingWindowAnalyzer::Aggregate( @@ -31,11 +73,7 @@ IoPatternSnapshot SlidingWindowAnalyzer::Aggregate( Append(current); std::lock_guard lock(mutex_); IoPatternSnapshot aggregate = current; - aggregate.keys.clear(); - for (const auto& snapshot : history_) { - aggregate.keys.insert(aggregate.keys.end(), snapshot.keys.begin(), - snapshot.keys.end()); - } + aggregate.keys = LatestKeys(history_); return aggregate; } @@ -67,14 +105,12 @@ WorkloadFeatureStats SlidingWindowAnalyzer::FeatureStats() const { std::lock_guard lock(mutex_); std::vector tokens, fanouts, matches, frequencies; std::vector blocks; - for (const auto& snapshot : history_) { - for (const auto& key : snapshot.keys) { - tokens.push_back(key.token_count); - fanouts.push_back(key.prefix_fanout); - matches.push_back(key.match_length); - frequencies.push_back(static_cast(key.access_count_window)); - blocks.push_back(key.block_size); - } + for (const auto& key : LatestKeys(history_)) { + tokens.push_back(key.token_count); + fanouts.push_back(key.prefix_fanout); + matches.push_back(key.match_length); + frequencies.push_back(static_cast(key.access_count_window)); + blocks.push_back(key.block_size); } const auto p90 = [](size_t size) { return size == 0 ? 0 : (size * 9) / 10; }; WorkloadFeatureStats stats; diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 8ce9aadc55..c1184dd941 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -160,6 +160,19 @@ DEFINE_int32(rpc_conn_timeout_seconds, 0, "Connection timeout in seconds (0 = no timeout)"); DEFINE_bool(rpc_enable_tcp_no_delay, true, "Enable TCP_NODELAY for RPC connections"); +DEFINE_string(io_pattern_cfm_endpoint, "", + "Central CFM Master RPC endpoint (host:port); empty serves CFM " + "requests without outbound reporting"); +DEFINE_string(io_pattern_cfm_node_id, "", + "Stable node id used for CFM policy polling; defaults to cluster_id"); +DEFINE_string(io_pattern_cfm_auth_token, "", + "Authentication token for CFM node report/poll RPCs"); +DEFINE_string(io_pattern_cfm_producer_auth_token, "", + "Separate token authorized to enqueue CFM policies"); +DEFINE_uint32(io_pattern_cfm_timeout_ms, 500, + "CFM RPC request timeout in milliseconds"); +DEFINE_uint32(io_pattern_cfm_policy_queue_capacity, 4096, + "Maximum queued CFM policies per node"); DEFINE_validator(eviction_ratio, [](const char* flagname, double value) { if (value < 0.0 || value > 1.0) { LOG(FATAL) << "Mem eviction ratio must be between 0.0 and 1.0"; @@ -483,6 +496,26 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, default_config.GetBool("rpc_enable_tcp_no_delay", &master_config.rpc_enable_tcp_no_delay, FLAGS_rpc_enable_tcp_no_delay); + default_config.GetString("io_pattern_cfm_endpoint", + &master_config.io_pattern_cfm.endpoint, + FLAGS_io_pattern_cfm_endpoint); + default_config.GetString("io_pattern_cfm_node_id", + &master_config.io_pattern_cfm.node_id, + FLAGS_io_pattern_cfm_node_id); + default_config.GetString("io_pattern_cfm_auth_token", + &master_config.io_pattern_cfm.auth_token, + FLAGS_io_pattern_cfm_auth_token); + default_config.GetString( + "io_pattern_cfm_producer_auth_token", + &master_config.io_pattern_cfm.producer_auth_token, + FLAGS_io_pattern_cfm_producer_auth_token); + default_config.GetUInt32("io_pattern_cfm_timeout_ms", + &master_config.io_pattern_cfm.timeout_ms, + FLAGS_io_pattern_cfm_timeout_ms); + default_config.GetUInt32( + "io_pattern_cfm_policy_queue_capacity", + &master_config.io_pattern_cfm.policy_queue_capacity, + FLAGS_io_pattern_cfm_policy_queue_capacity); default_config.GetDurationMs("default_kv_lease_ttl", &master_config.default_kv_lease_ttl, mooncake::DEFAULT_DEFAULT_KV_LEASE_TTL); @@ -779,6 +812,41 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, } google::CommandLineFlagInfo info; + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_endpoint", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.endpoint = FLAGS_io_pattern_cfm_endpoint; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_node_id", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.node_id = FLAGS_io_pattern_cfm_node_id; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_auth_token", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.auth_token = + FLAGS_io_pattern_cfm_auth_token; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_cfm_producer_auth_token", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.producer_auth_token = + FLAGS_io_pattern_cfm_producer_auth_token; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_timeout_ms", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.timeout_ms = FLAGS_io_pattern_cfm_timeout_ms; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_cfm_policy_queue_capacity", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.policy_queue_capacity = + FLAGS_io_pattern_cfm_policy_queue_capacity; + } if ((google::GetCommandLineFlagInfo("enable_cxl", &info) && !info.is_default) || !conf_set) { @@ -1477,6 +1545,29 @@ int main(int argc, char* argv[]) { << ", must be 'cachelib' or 'offset'"; return 1; } + if (!master_config.io_pattern_cfm.endpoint.empty() && + master_config.io_pattern_cfm.auth_token.empty()) { + LOG(FATAL) << "io_pattern_cfm_auth_token is required when " + "io_pattern_cfm_endpoint is configured"; + return 1; + } + if (!master_config.io_pattern_cfm.producer_auth_token.empty() && + master_config.io_pattern_cfm.producer_auth_token == + master_config.io_pattern_cfm.auth_token) { + LOG(FATAL) << "io_pattern_cfm_producer_auth_token must differ from " + "io_pattern_cfm_auth_token"; + return 1; + } + if (master_config.io_pattern_cfm.timeout_ms == 0 || + master_config.io_pattern_cfm.timeout_ms > 10'000 || + master_config.io_pattern_cfm.policy_queue_capacity == 0) { + LOG(FATAL) << "io_pattern_cfm_timeout_ms must be in [1, 10000] and " + "io_pattern_cfm_policy_queue_capacity must be non-zero"; + return 1; + } + if (master_config.io_pattern_cfm.node_id.empty()) { + master_config.io_pattern_cfm.node_id = master_config.cluster_id; + } const char* value = std::getenv("MC_RPC_PROTOCOL"); std::string protocol = "tcp"; @@ -1546,6 +1637,14 @@ int main(int argc, char* argv[]) { << ", client_ttl=" << master_config.client_live_ttl_sec << ", rpc_thread_num=" << master_config.rpc_thread_num << ", rpc_port=" << master_config.rpc_port + << ", io_pattern_cfm_endpoint=" + << (master_config.io_pattern_cfm.endpoint.empty() + ? "" + : master_config.io_pattern_cfm.endpoint) + << ", io_pattern_cfm_node_id=" + << master_config.io_pattern_cfm.node_id + << ", io_pattern_cfm_server_enabled=" + << !master_config.io_pattern_cfm.auth_token.empty() << ", rpc_address=" << master_config.rpc_address << ", rpc_interface=" << master_config.rpc_interface << ", rpc_conn_timeout_seconds=" diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 49f8be3c10..89c383818c 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -58,6 +58,11 @@ #include "ha_metric_manager.h" #include "metadata_store.h" #include "io_pattern/runtime.h" +#include "io_pattern/cfm_client_impl.h" +#include "io_pattern/cfm_protocol.h" +#include "io_pattern/cfm_service.h" +#include "io_pattern/resilient_cfm_channel.h" +#include "io_pattern/rpc_transport.h" namespace mooncake { @@ -413,19 +418,74 @@ MasterService::MasterService(const MasterServiceConfig& config) << ")"; } - io_pattern_runtime_ = std::make_unique( + io_pattern::IoPatternRuntime::Config io_pattern_config; + std::shared_ptr cfm_rpc_channel; + if (!config.io_pattern_cfm.endpoint.empty()) { + if (config.io_pattern_cfm.timeout_ms == 0 || + config.io_pattern_cfm.timeout_ms > 10'000) { + throw std::invalid_argument( + "io_pattern_cfm_timeout_ms must be in [1, 10000]"); + } + if (config.io_pattern_cfm.auth_token.empty()) { + throw std::invalid_argument( + "io_pattern_cfm_auth_token is required when " + "io_pattern_cfm_endpoint is configured"); + } + const std::string node_id = config.io_pattern_cfm.node_id.empty() + ? config.cluster_id + : config.io_pattern_cfm.node_id; + auto transport = std::make_shared( + config.io_pattern_cfm.endpoint, node_id, + std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms)); + if (!transport->Authenticate(config.io_pattern_cfm.auth_token)) { + throw std::runtime_error( + "failed to authenticate with configured IO-pattern CFM " + "endpoint " + + config.io_pattern_cfm.endpoint); + } + cfm_rpc_channel = std::make_shared( + std::move(transport), + std::make_shared(), + io_pattern::CfmRpcConfig{ + .timeout = + std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms), + .auth_token = config.io_pattern_cfm.auth_token}); + io_pattern_config.report_sink = + io_pattern::MakeCfmMetricBatchSink(cfm_rpc_channel); + io_pattern_cfm_channel_ = + std::make_shared(cfm_rpc_channel); + } + + io_pattern_runtime_ = std::make_shared( io_pattern::IoPatternRuntime::Handlers{ .eviction = [this](const io_pattern::EvictionPlan& plan) { - bool evicted = plan.candidates.empty(); - std::unordered_map targets; + struct TenantCandidates { + uint64_t bytes{0}; + std::unordered_set keys; + }; + if (plan.target_bytes == 0) return ErrorCode::OK; + if (plan.candidates.empty()) return ErrorCode::OBJECT_NOT_FOUND; + uint64_t total_freed = 0; + std::unordered_map + targets; for (const auto& candidate : plan.candidates) { - targets[candidate.object.tenant_id] += candidate.bytes; + auto& target = targets[candidate.object.tenant_id]; + target.bytes += candidate.bytes; + target.keys.insert(candidate.object.key); } - for (const auto& [tenant, bytes] : targets) { - const auto result = EvictTenantMemoryForQuota(tenant, bytes); - evicted = evicted || result.freed_bytes != 0; + for (const auto& [tenant, target] : targets) { + const auto result = EvictTenantMemoryForQuota( + tenant, target.bytes, &target.keys); + total_freed = + result.freed_bytes > + std::numeric_limits::max() - + total_freed + ? std::numeric_limits::max() + : total_freed + result.freed_bytes; } - return evicted ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; + return total_freed >= plan.target_bytes + ? ErrorCode::OK + : ErrorCode::OBJECT_NOT_FOUND; }, .prefetch = [this](const io_pattern::PrefetchPlan& plan) { for (const auto& candidate : plan.candidates) { @@ -456,7 +516,12 @@ MasterService::MasterService(const MasterServiceConfig& config) PromotionQueueResult::kQueued ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; - }}); + }}, + std::move(io_pattern_config)); + io_pattern_cfm_service_ = std::make_shared( + io_pattern_runtime_, config.io_pattern_cfm.auth_token, + config.io_pattern_cfm.policy_queue_capacity, + config.io_pattern_cfm.producer_auth_token); kv_event_publisher_ = std::make_unique(BuildKvEventConfig(config)); @@ -575,6 +640,34 @@ MasterService::MasterService(const MasterServiceConfig& config) segment_manager_.initializeCxlAllocator(cxl_path_, cxl_size_); VLOG(1) << "action=start_cxl_global_allocator"; } + + // Start the CFM consumer last. If any preceding initialization throws, + // constructor unwinding must not encounter a joinable std::thread. + if (io_pattern_cfm_channel_) { + io_pattern_cfm_client_ = std::make_unique( + io_pattern_cfm_channel_, + [this](const io_pattern::PolicyCommand& command) { + return io_pattern_runtime_ + ? io_pattern_runtime_->ExecuteCommand(command) + : ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + }); + io_pattern_cfm_polling_ = true; + io_pattern_cfm_poll_thread_ = std::thread([this] { + while (io_pattern_cfm_polling_.load(std::memory_order_acquire)) { + const auto result = + io_pattern_cfm_client_->PollAndDispatchPolicy(); + std::unique_lock lock(io_pattern_cfm_poll_mutex_); + io_pattern_cfm_poll_cv_.wait_for( + lock, + result == ErrorCode::OK ? std::chrono::milliseconds(100) + : std::chrono::seconds(1), + [this] { + return !io_pattern_cfm_polling_.load( + std::memory_order_acquire); + }); + } + }); + } } std::unique_ptr @@ -614,6 +707,14 @@ MasterService::CreateSnapshotCatalogStore() { } MasterService::~MasterService() { + io_pattern_cfm_polling_.store(false, std::memory_order_release); + io_pattern_cfm_poll_cv_.notify_all(); + if (io_pattern_cfm_poll_thread_.joinable()) { + io_pattern_cfm_poll_thread_.join(); + } + io_pattern_cfm_client_.reset(); + io_pattern_cfm_channel_.reset(); + if (ordered_oplog_writer_) { ordered_oplog_writer_->Stop(); } @@ -660,6 +761,11 @@ MasterService::~MasterService() { job_dispatch_thread_.join(); } + // Its admission worker executes handlers that capture this service. Stop + // and join it while all handler dependencies are still alive. + io_pattern_cfm_service_.reset(); + io_pattern_runtime_.reset(); + // Reset snapshot manager after all other threads have joined // This triggers the destructor which joins the snapshot thread if (snapshot_manager_) { @@ -4052,8 +4158,10 @@ auto MasterService::PutStart(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::TENANT_QUOTA_EXCEEDED); } -auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, - const TenantId& tenant_id, ReplicaType replica_type) +auto MasterService::PutEndInternal( + const UUID& client_id, const ObjectMeta& object_meta, + const TenantId& tenant_id, ReplicaType replica_type, + uint32_t write_batch_size, bool overwrite) -> tl::expected { const auto& key = object_meta.key; std::shared_lock shared_lock(snapshot_mutex_); @@ -4170,7 +4278,12 @@ auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, : io_pattern::CacheTier::kL3NofSsd, .operation = io_pattern::IoOperation::kPut, .is_hit = true, - .write_batch_size = 1}); + .write_batch_size = write_batch_size, + .overwrite = overwrite}); + if (replica_type != ReplicaType::MEMORY) { + io_pattern_runtime_->ScheduleAdmission( + {object_id.tenant_id, key}, io_pattern::CacheTier::kL1Host); + } } if (enable_oplog_ && ordered_oplog_writer_) { @@ -4430,6 +4543,14 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, return {}; } +auto MasterService::PutEnd(const UUID& client_id, + const ObjectMeta& object_meta, + const TenantId& tenant_id, ReplicaType replica_type) + -> tl::expected { + return PutEndInternal(client_id, object_meta, tenant_id, replica_type, + /*write_batch_size=*/1, /*overwrite=*/false); +} + auto MasterService::PutEnd(const UUID& client_id, const std::string& key, const TenantId& tenant_id, ReplicaType replica_type) -> tl::expected { @@ -4444,8 +4565,11 @@ std::vector> MasterService::BatchPutEnd( std::vector> results; results.reserve(object_metas.size()); for (const auto& object_meta : object_metas) { - results.emplace_back( - PutEnd(client_id, object_meta, tenant_id, replica_type)); + results.emplace_back(PutEndInternal( + client_id, object_meta, tenant_id, replica_type, + static_cast(std::min( + object_metas.size(), std::numeric_limits::max())), + /*overwrite=*/false)); } return results; } @@ -4823,7 +4947,8 @@ auto MasterService::UpsertEnd(const UUID& client_id, const TenantId& tenant_id, ReplicaType replica_type) -> tl::expected { - return PutEnd(client_id, object_meta, tenant_id, replica_type); + return PutEndInternal(client_id, object_meta, tenant_id, replica_type, + /*write_batch_size=*/1, /*overwrite=*/true); } auto MasterService::UpsertEnd(const UUID& client_id, const std::string& key, @@ -4878,7 +5003,18 @@ MasterService::BatchUpsertStart(const UUID& client_id, std::vector> MasterService::BatchUpsertEnd( const UUID& client_id, const std::vector& object_metas, const TenantId& tenant_id) { - return BatchPutEnd(client_id, object_metas, tenant_id, ReplicaType::ALL); + assert(tenant_id.IsValid()); + std::vector> results; + results.reserve(object_metas.size()); + const auto batch_size = static_cast( + std::min(object_metas.size(), + std::numeric_limits::max())); + for (const auto& object_meta : object_metas) { + results.emplace_back(PutEndInternal( + client_id, object_meta, tenant_id, ReplicaType::ALL, batch_size, + /*overwrite=*/true)); + } + return results; } std::vector> MasterService::BatchUpsertRevoke( @@ -7536,12 +7672,16 @@ void MasterService::EvictionThreadFunc() { .memory_used_ratio = static_cast(used_ratio)}); const auto capacity = std::max( 0, MasterMetricManager::instance().get_total_mem_capacity()); - io_pattern_runtime_->Execute( + const auto status = io_pattern_runtime_->Execute( io_pattern::CacheTier::kL1Host, static_cast(evict_ratio_target * capacity), {}); + if (status.eviction != ErrorCode::OK) { + BatchEvict(evict_ratio_target, evict_ratio_lowerbound); + } + } else { + BatchEvict(evict_ratio_target, evict_ratio_lowerbound); } - BatchEvict(evict_ratio_target, evict_ratio_lowerbound); - LOG(INFO) << "[EVICT-DONE] BatchEvict execution completed."; + LOG(INFO) << "[EVICT-DONE] eviction execution completed."; last_discard_time = now; } else if (now - last_discard_time > put_start_release_timeout_sec_) { // Try discarding expired processing keys and ongoing replication @@ -8140,7 +8280,9 @@ tl::expected MasterService::ApplySnapshotState( MasterService::TenantQuotaEvictionResult MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, - uint64_t target_bytes) { + uint64_t target_bytes, + const std::unordered_set* + candidate_keys) { TenantQuotaEvictionResult total; if (target_bytes == 0) { return total; @@ -8257,6 +8399,17 @@ MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, .evicted_objects = freed > 0 ? 1U : 0U}; } + // Group eviction is atomic. A policy plan naming only part of a group + // must not silently expand into unplanned objects; let the caller take + // the legacy fallback path instead. + if (candidate_keys && + std::any_of(group_it->second.begin(), group_it->second.end(), + [candidate_keys](const std::string& member_key) { + return !candidate_keys->contains(member_key); + })) { + return {}; + } + for (const auto& member_key : group_it->second) { auto member_it = tenant_state.metadata.find(member_key); if (member_it != tenant_state.metadata.end() && @@ -8311,6 +8464,10 @@ MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, for (auto it = tenant_state.metadata.begin(); it != tenant_state.metadata.end() && total.freed_bytes < target_bytes;) { + if (candidate_keys && !candidate_keys->contains(it->first)) { + ++it; + continue; + } auto& metadata = it->second; if (metadata.IsHardPinned() || !metadata.IsLeaseExpired(now) || diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index e3b7891086..3ddc66d936 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -100,7 +100,8 @@ WrappedMasterService::WrappedMasterService( const WrappedMasterServiceConfig& config, HttpMetadataServer* http_metadata_server, const std::string& http_metadata_remote_url) - : master_service_(MasterServiceConfig(config)) { + : master_service_(MasterServiceConfig(config)), + cfm_rpc_service_(master_service_.GetCfmService()) { // Configure metadata cleanup on client timeout. Prefer the co-located // in-process server; otherwise fall back to a separately-deployed HTTP // metadata server derived from the cluster configuration. @@ -1836,6 +1837,12 @@ void RegisterRpcService( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::PutStart>( &wrapped_master_service); + auto& cfm = wrapped_master_service.CfmRpcEndpoint(); + server.register_handler<&io_pattern::CfmRpcService::Authenticate>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::Send>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::Receive>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::Acknowledge>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::EnqueuePolicy>(&cfm); server.register_handler<&mooncake::WrappedMasterService::PutEnd>( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::PutRevoke>( diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index b5836d306f..51f5329d09 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -3,12 +3,16 @@ #include "io_pattern/policy_strategies.h" #include +#include +#include +#include #include #include #include #include #include +#include namespace mooncake::io_pattern { namespace { @@ -109,12 +113,23 @@ class TestCfmChannel final : public CfmChannel { snapshot = value; return send_ok; } - std::optional PollPolicy() override { return policy; } + CfmPollResult PollPolicyResult() override { + return policy ? CfmPollResult::Command(*policy, 42) + : CfmPollResult::Empty(); + } + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override { + acknowledged_delivery_id = delivery_id; + acknowledged_success = success; + return acknowledge_ok; + } ErrorCode ExecutePrefetch(const PrefetchPlan& value) override { plan = value; return execute_code; } bool send_ok{true}; + bool acknowledge_ok{true}; + bool acknowledged_success{false}; + uint64_t acknowledged_delivery_id{0}; ErrorCode execute_code{ErrorCode::OK}; IoPatternSnapshot snapshot; std::optional policy; @@ -126,7 +141,9 @@ class FlakyCfmChannel final : public CfmChannel { bool SendSnapshot(const IoPatternSnapshot&) override { return send_failures-- <= 0; } - std::optional PollPolicy() override { return PrefetchPlan{}; } + CfmPollResult PollPolicyResult() override { + return CfmPollResult::Command(PrefetchPlan{}); + } ErrorCode ExecutePrefetch(const PrefetchPlan&) override { return ErrorCode::RPC_FAIL; } @@ -142,13 +159,24 @@ class TestRpcTransport final : public CfmRpcTransport { last_timeout = timeout; return send_ok; } - std::optional Receive(std::string_view method, - std::chrono::milliseconds timeout) override { + CfmReceiveResult Receive(std::string_view method, + std::chrono::milliseconds timeout) override { last_method = std::string(method); last_timeout = timeout; - return response; + return response ? CfmReceiveResult::Payload(*response) + : CfmReceiveResult::Empty(); + } + bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds timeout) override { + acknowledged_delivery_id = delivery_id; + acknowledged_success = success; + last_timeout = timeout; + return acknowledge_ok; } bool send_ok{true}; + bool acknowledge_ok{true}; + bool acknowledged_success{false}; + uint64_t acknowledged_delivery_id{0}; std::optional response; std::string last_method; std::string last_payload; @@ -339,6 +367,65 @@ TEST(IoPatternFrameworkTest, CollectorImplDerivesWritePathMetrics) { EXPECT_TRUE(key.write_burst); } +TEST(IoPatternFrameworkTest, CollectorImplBoundsAccessCountByTimeWindow) { + uint64_t now_ns = 10; + IoPatternCollectorImpl collector(IoPatternCollectorImpl::Config{ + .access_window_ns = 100, + .access_bucket_ns = 1, + .now_ns = [&] { return now_ns; }}); + AccessRecord access{.object = {TenantId("tenant-a"), "key"}, + .observed_at_ns = 10}; + collector.RecordAccess(access.object.key, access); + access.observed_at_ns = 50; + collector.RecordAccess(access.object.key, access); + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 2); + + access.observed_at_ns = 111; + now_ns = 111; + collector.RecordAccess(access.object.key, access); + // A true sliding window retains the event at 50 even though the first + // event's fixed 10..110 bucket has ended. + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 2); + + now_ns = 212; + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 0); +} + +TEST(IoPatternFrameworkTest, CollectorExpiresMergedAccessWindowsLocally) { + uint64_t now_ns = 10; + IoPatternCollectorImpl collector(IoPatternCollectorImpl::Config{ + .access_window_ns = 100, + .access_bucket_ns = 1, + .now_ns = [&] { return now_ns; }}); + IoPatternSnapshot remote; + remote.keys.push_back( + KeyMetrics{.object = {TenantId("tenant-a"), "remote"}, + .access_count_window = 7, + .write_frequency = 3}); + + collector.MergeSnapshot(remote); + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 7); + now_ns = 111; + const auto expired = collector.GetSnapshot().keys.front(); + EXPECT_EQ(expired.access_count_window, 0); + EXPECT_EQ(expired.write_frequency, 0); +} + +TEST(IoPatternFrameworkTest, CollectorHardCapsBucketsForOutOfOrderInput) { + uint64_t now_ns = 3; + IoPatternCollectorImpl collector(IoPatternCollectorImpl::Config{ + .access_window_ns = 1'000, + .access_bucket_ns = 1, + .max_access_buckets_per_key = 2, + .now_ns = [&] { return now_ns; }}); + AccessRecord access{.object = {TenantId("tenant-a"), "key"}}; + for (uint64_t timestamp : {1ULL, 2ULL, 3ULL}) { + access.observed_at_ns = timestamp; + collector.RecordAccess(access.object.key, access); + } + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 2); +} + TEST(IoPatternFrameworkTest, ThresholdAnalyzerClassifiesDocumentedWorkloads) { ThresholdAnalyzer analyzer; IoPatternSnapshot code_agent; @@ -460,6 +547,17 @@ TEST(IoPatternFrameworkTest, PrefixAdmissionUsesTierSpecificSignals) { const auto rejected = admission.Evaluate(key.object, CacheTier::kL0Hbm, context); EXPECT_EQ(rejected.decision, AdmissionDecision::kRejectPrefix); + + context.snapshot.storage = { + StorageMetric{.source_id = "ssd", + .tier = CacheTier::kL3NofSsd, + .memory_used_ratio = 0.1F}, + StorageMetric{.source_id = "host", + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.95F}}; + EXPECT_EQ(admission.Evaluate(key.object, CacheTier::kL1Host, context) + .decision, + AdmissionDecision::kRejectWatermark); } TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { @@ -667,14 +765,21 @@ TEST(IoPatternFrameworkTest, ReporterBatchesBoundsAndCountsDrops) { TEST(IoPatternFrameworkTest, ReporterAdaptsFlushIntervalToLoad) { IoPatternReporter reporter(4, [](const MetricBatch&) { return true; }); + reporter.UpdateLoad(0.25F, 0); EXPECT_EQ(reporter.RecommendedFlushInterval(), - std::chrono::milliseconds(1000)); - reporter.Enqueue(InferenceMetrics{}); + std::chrono::milliseconds(100)); + reporter.UpdateLoad(0.50F, 0); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(200)); + reporter.UpdateLoad(0.80F, 0); EXPECT_EQ(reporter.RecommendedFlushInterval(), std::chrono::milliseconds(500)); - reporter.Enqueue(InferenceMetrics{}); + reporter.UpdateLoad(0.95F, 0); EXPECT_EQ(reporter.RecommendedFlushInterval(), - std::chrono::milliseconds(100)); + std::chrono::milliseconds(1000)); + reporter.UpdateLoad(0.25F, 101'000); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(1000)); } TEST(IoPatternFrameworkTest, ReporterEnforcesPerTenantFairness) { @@ -722,6 +827,11 @@ TEST(IoPatternFrameworkTest, CfmClientDispatchesReceivedPolicyCommands) { channel->policy = PolicyCommand{AdmissionResult{}}; EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); EXPECT_EQ(dispatched, 2); + EXPECT_EQ(channel->acknowledged_delivery_id, 42); + EXPECT_TRUE(channel->acknowledged_success); + channel->policy.reset(); + EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); + EXPECT_EQ(dispatched, 2); } TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { @@ -739,6 +849,17 @@ TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { EXPECT_TRUE(channel.degraded()); } +TEST(IoPatternFrameworkTest, EmptyPolicyPollKeepsChannelHealthy) { + auto idle = std::make_shared(); + ResilientCfmChannel channel( + idle, CfmRetryConfig{.max_retries = 2, .degrade_after_failures = 2}); + + EXPECT_FALSE(channel.PollPolicy().has_value()); + EXPECT_FALSE(channel.PollPolicy().has_value()); + EXPECT_EQ(channel.consecutive_failures(), 0); + EXPECT_FALSE(channel.degraded()); +} + TEST(IoPatternFrameworkTest, ResilientAnalyzerFallsBackAfterFailure) { ResilientAnalyzer analyzer(std::make_shared(), 2); EXPECT_EQ(analyzer.DetectWorkloadType({}), WorkloadType::kMixed); @@ -853,6 +974,39 @@ TEST(IoPatternFrameworkTest, InProcessCfmTransportAuthenticatesAndDispatches) { EXPECT_FALSE(unauthorized.SendSnapshot({})); } +TEST(IoPatternFrameworkTest, InProcessTransportDoesNotHoldLockAcrossHandler) { + std::promise handler_entered; + std::promise release_handler; + auto release = release_handler.get_future().share(); + auto transport = std::make_shared( + "shared-secret", [&](std::string_view, std::string_view) { + handler_entered.set_value(); + release.wait(); + return true; + }); + ASSERT_TRUE(transport->Authenticate("shared-secret")); + + std::thread sender([&] { + EXPECT_TRUE(transport->Send("report_snapshot", {}, + std::chrono::milliseconds(10))); + }); + if (handler_entered.get_future().wait_for(std::chrono::seconds(1)) != + std::future_status::ready) { + release_handler.set_value(); + sender.join(); + FAIL() << "send handler did not start"; + return; + } + auto enqueue = std::async(std::launch::async, [&] { + transport->EnqueuePolicy("policy"); + return true; + }); + EXPECT_EQ(enqueue.wait_for(std::chrono::milliseconds(100)), + std::future_status::ready); + release_handler.set_value(); + sender.join(); +} + TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { auto runtime = std::make_shared( IoPatternRuntime::Handlers{.eviction = [](const EvictionPlan&) { @@ -881,6 +1035,158 @@ TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { EXPECT_EQ(snapshot.keys.front().access_count_window, 1); } +TEST(IoPatternFrameworkTest, CfmServiceAuthenticatesAndBoundsPolicyQueues) { + int admissions = 0; + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [&admissions](const AdmissionResult&) { + ++admissions; + return ErrorCode::OK; + }}); + CfmService service(runtime, "secret", 1, "producer"); + CfmBinaryCodec codec; + + EXPECT_FALSE(service.Authenticate("wrong")); + EXPECT_TRUE(service.Authenticate("secret")); + EXPECT_FALSE(service.EnqueuePolicy("node-a", "first", "secret")); + const auto admission = codec.EncodePolicy(AdmissionResult{ + .object = {TenantId("tenant"), "key"}, + .target_tier = CacheTier::kL1Host, + .decision = AdmissionDecision::kAdmit}); + const auto second = codec.EncodePolicy(PrefetchPlan{}); + EXPECT_FALSE(service.EnqueuePolicy("node-a", "malformed", "producer")); + EXPECT_TRUE(service.EnqueuePolicy("node-a", admission, "producer")); + EXPECT_FALSE(service.EnqueuePolicy("node-a", second, "producer")); + const auto delivery = service.PollPolicy("node-a", "secret"); + ASSERT_TRUE(delivery.has_value()); + EXPECT_EQ(delivery->second, admission); + EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, false, + "secret")); + EXPECT_TRUE(service.PollPolicy("node-a", "secret").has_value()); + EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, true, + "secret")); + EXPECT_FALSE(service.PollPolicy("node-a", "secret").has_value()); + + EXPECT_FALSE(service.Send("", "execute_policy", admission, "secret")); + EXPECT_FALSE( + service.Send("node-a", "execute_policy", admission, "secret")); + EXPECT_TRUE( + service.Send("node-a", "execute_policy", admission, "producer")); + EXPECT_EQ(admissions, 1); +} + +TEST(IoPatternFrameworkTest, CoroRpcCfmTransportRunsTheProductionWirePath) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + auto service = + std::make_shared(runtime, "secret", 2, "producer"); + CfmRpcService endpoint(service); + coro_rpc::coro_rpc_server server(1, 0, "127.0.0.1"); + server.register_handler<&CfmRpcService::Authenticate>(&endpoint); + server.register_handler<&CfmRpcService::Send>(&endpoint); + server.register_handler<&CfmRpcService::Receive>(&endpoint); + server.register_handler<&CfmRpcService::Acknowledge>(&endpoint); + server.register_handler<&CfmRpcService::EnqueuePolicy>(&endpoint); + ASSERT_FALSE(server.async_start().hasResult()); + + const auto rejected_poll = + endpoint.Receive("poll_policy", "node-a", "wrong"); + EXPECT_FALSE(rejected_poll.first); + const auto empty_poll = endpoint.Receive("poll_policy", "node-a", "secret"); + EXPECT_TRUE(empty_poll.first); + EXPECT_FALSE(empty_poll.second.has_value()); + + CoroRpcCfmTransport transport( + "127.0.0.1:" + std::to_string(server.port()), "node-a", + std::chrono::milliseconds(500)); + EXPECT_FALSE(transport.Authenticate("wrong")); + ASSERT_TRUE(transport.Authenticate("secret")); + + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "remote-key"}, + .is_hit = true}); + batch.storage.push_back(StorageMetric{.source_id = "spoofed", + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.75F}); + EXPECT_TRUE(transport.Send("report_metric_batch", + codec.EncodeMetricBatch(batch), + std::chrono::milliseconds(500))); + const auto snapshot = runtime->Snapshot(); + ASSERT_EQ(snapshot.keys.size(), 1); + ASSERT_EQ(snapshot.storage.size(), 1); + EXPECT_EQ(snapshot.keys.front().object.key, "remote-key"); + EXPECT_EQ(snapshot.storage.front().source_id, "node-a"); + + const auto policy = codec.EncodePolicy(PrefetchPlan{}); + CoroRpcCfmTransport producer( + "127.0.0.1:" + std::to_string(server.port()), "producer", + std::chrono::milliseconds(500)); + ASSERT_TRUE(producer.Authenticate("producer")); + EXPECT_TRUE(producer.EnqueuePolicy("node-a", policy, + std::chrono::milliseconds(500))); + const auto received = + transport.Receive("poll_policy", std::chrono::milliseconds(500)); + EXPECT_EQ(received.status, CfmReceiveResult::Status::kPayload); + EXPECT_EQ(received.payload, policy); + EXPECT_NE(received.delivery_id, 0); + EXPECT_TRUE(transport.Acknowledge(received.delivery_id, false, + std::chrono::milliseconds(500))); + const auto redelivered = + transport.Receive("poll_policy", std::chrono::milliseconds(500)); + EXPECT_EQ(redelivered.delivery_id, received.delivery_id); + EXPECT_EQ(redelivered.payload, policy); + EXPECT_TRUE(transport.Acknowledge(redelivered.delivery_id, true, + std::chrono::milliseconds(500))); + EXPECT_EQ(transport.Receive("poll_policy", std::chrono::milliseconds(500)) + .status, + CfmReceiveResult::Status::kEmpty); + server.stop(); +} + +TEST(IoPatternFrameworkTest, CfmProducesNodePolicyFromHighWatermarkReport) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + CfmService service(runtime, "node-secret", 8, "producer-secret"); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "cold-key"}, + .block_size = 1024, + .tier = CacheTier::kL1Host, + .is_hit = false}); + batch.storage.push_back(StorageMetric{.tier = CacheTier::kL1Host, + .used_bytes = 950, + .capacity_bytes = 1000, + .memory_used_ratio = 0.95F}); + ASSERT_TRUE(service.Send("node-a", "report_metric_batch", + codec.EncodeMetricBatch(batch), "node-secret")); + + std::optional> delivery; + for (size_t attempt = 0; attempt < 100 && !delivery; ++attempt) { + delivery = service.PollPolicy("node-a", "node-secret"); + if (!delivery) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + ASSERT_TRUE(delivery.has_value()); + const auto command = codec.DecodePolicy(delivery->second); + ASSERT_TRUE(command.has_value()); + const auto* eviction = std::get_if(&*command); + ASSERT_NE(eviction, nullptr); + ASSERT_EQ(eviction->candidates.size(), 1); + EXPECT_EQ(eviction->candidates.front().object.key, "cold-key"); + EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, true, + "node-secret")); +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { @@ -972,14 +1278,16 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { SlidingWindowAnalyzer analyzer(100); IoPatternSnapshot first; first.generated_at_ns = 10; - first.keys.push_back(KeyMetrics{.token_count = 20 * 1024, + first.keys.push_back(KeyMetrics{.object = {TenantId("tenant"), "first"}, + .token_count = 20 * 1024, .prefix_fanout = 20, .match_length = 512, .block_size = 100, .access_count_window = 1}); IoPatternSnapshot second; second.generated_at_ns = 50; - second.keys.push_back(KeyMetrics{.token_count = 30, + second.keys.push_back(KeyMetrics{.object = {TenantId("tenant"), "second"}, + .token_count = 30, .prefix_fanout = 20, .match_length = 300, .block_size = 300, @@ -992,6 +1300,39 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { EXPECT_EQ(stats.block_p90, 300); } +TEST(IoPatternFrameworkTest, SlidingWindowDeduplicatesObjectsAndBoundsHistory) { + SlidingWindowAnalyzer analyzer(1'000, {}, 2); + const ObjectRef object{TenantId("tenant-a"), "same-key"}; + for (uint64_t timestamp = 1; timestamp <= 3; ++timestamp) { + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = timestamp; + snapshot.keys.push_back(KeyMetrics{.object = object, + .token_count = + static_cast(timestamp), + .access_count_window = timestamp}); + analyzer.Analyze(snapshot); + } + + const auto stats = analyzer.FeatureStats(); + EXPECT_EQ(stats.samples, 1); + EXPECT_EQ(stats.token_median, 3); + EXPECT_EQ(stats.frequency_median, 3); +} + +TEST(IoPatternFrameworkTest, SlidingWindowCapsASingleOversizedSnapshot) { + SlidingWindowAnalyzer analyzer(1'000, {}, 2); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = 1; + snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant"), "c"}}, + KeyMetrics{.object = {TenantId("tenant"), "a"}}, + KeyMetrics{.object = {TenantId("tenant"), "b"}}, + }; + + analyzer.Analyze(snapshot); + EXPECT_EQ(analyzer.FeatureStats().samples, 2); +} + TEST(IoPatternFrameworkTest, KMeansFallbackLabelsIndependentSessions) { SlidingWindowAnalyzer analyzer; IoPatternSnapshot snapshot; @@ -1059,6 +1400,21 @@ TEST(IoPatternFrameworkTest, LegacyEvictionAdapterUsesLruFallback) { EXPECT_EQ(plan.candidates.front().object.key, "first"); } +TEST(IoPatternFrameworkTest, ScoreBasedEvictionMayCrossTheByteTarget) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "large"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}}; + context.analysis.keys = { + KeyPattern{.object = {TenantId("tenant-a"), "large"}}}; + ScoreBasedEvictionOps eviction; + + const auto plan = eviction.Evaluate(context, CacheTier::kL1Host, 32); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().bytes, 64); +} + TEST(IoPatternFrameworkTest, ScoreBasedEvictionUsesTierSpecificSignals) { PolicyContext context; context.snapshot.keys = { @@ -1193,5 +1549,26 @@ TEST(IoPatternFrameworkTest, RuntimeExecutesCfmCommandsThroughStorageHandlers) { EXPECT_EQ(admissions, 1); } +TEST(IoPatternFrameworkTest, RuntimeSchedulesAdmissionOffTheProducerPath) { + std::promise handled; + IoPatternRuntime runtime( + {.eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [&handled](const AdmissionResult& result) { + handled.set_value(result); + return ErrorCode::OK; + }}); + AccessRecord access{.object = {TenantId("tenant"), "disk-key"}, + .tier = CacheTier::kL3NofSsd, + .operation = IoOperation::kPut}; + runtime.RecordAccess(access.object.key, access); + + EXPECT_TRUE(runtime.ScheduleAdmission(access.object, CacheTier::kL1Host)); + auto result = handled.get_future(); + ASSERT_EQ(result.wait_for(std::chrono::seconds(1)), + std::future_status::ready); + EXPECT_EQ(result.get().object, access.object); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/master_service_config_test.cpp b/mooncake-store/tests/master_service_config_test.cpp index 4b69bcf5f0..ed50c77325 100644 --- a/mooncake-store/tests/master_service_config_test.cpp +++ b/mooncake-store/tests/master_service_config_test.cpp @@ -46,4 +46,26 @@ TEST(MasterServiceConfigTest, OplogBatchMaxEntriesBuilderOverrideRespected) { EXPECT_EQ(17u, config.oplog_batch_max_entries); } +TEST(MasterServiceConfigTest, IoPatternCfmPropagatesToServingConfig) { + MasterConfig master_config{}; + master_config.io_pattern_cfm = {.endpoint = "cfm.example:50051", + .node_id = "master-a", + .auth_token = "secret", + .producer_auth_token = "producer-secret", + .timeout_ms = 750, + .policy_queue_capacity = 32}; + + MasterServiceSupervisorConfig supervisor_config(master_config); + WrappedMasterServiceConfig wrapped_config(supervisor_config, 1); + MasterServiceConfig service_config(wrapped_config); + + EXPECT_EQ(service_config.io_pattern_cfm.endpoint, "cfm.example:50051"); + EXPECT_EQ(service_config.io_pattern_cfm.node_id, "master-a"); + EXPECT_EQ(service_config.io_pattern_cfm.auth_token, "secret"); + EXPECT_EQ(service_config.io_pattern_cfm.producer_auth_token, + "producer-secret"); + EXPECT_EQ(service_config.io_pattern_cfm.timeout_ms, 750); + EXPECT_EQ(service_config.io_pattern_cfm.policy_queue_capacity, 32); +} + } // namespace mooncake::test