diff --git a/conanfile.py b/conanfile.py index cfd80a1c..aa998a00 100644 --- a/conanfile.py +++ b/conanfile.py @@ -10,7 +10,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "4.3.2" + version = "4.3.3" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeStore" diff --git a/src/lib/homestore_backend/gc_manager.cpp b/src/lib/homestore_backend/gc_manager.cpp index 40f289f7..1177e905 100644 --- a/src/lib/homestore_backend/gc_manager.cpp +++ b/src/lib/homestore_backend/gc_manager.cpp @@ -223,32 +223,52 @@ std::shared_ptr< GCManager::pdev_gc_actor > GCManager::get_pdev_gc_actor(uint32_ return it->second; } -float GCManager::get_chunk_gc_ratio(chunk_id_t chunk_id) { +GCManager::ChunkGCSnapshot GCManager::get_chunk_gc_snapshot(chunk_id_t chunk_id, uint8_t gc_thresh_low) { + ChunkGCSnapshot snap; auto chunk = m_chunk_selector->get_extend_vchunk(chunk_id); - // Only AVAILABLE chunks are eligible: INUSE means an open shard owns it, GC means already being processed. - if (chunk->m_state != ChunkState::AVAILABLE) { return 0.0f; } - - const auto defrag_blk_num = chunk->get_defrag_nblks(); - if (!defrag_blk_num) { return 0.0f; } - - // Chunks with no pg assignment are unowned and do not need GC. - if (!chunk->m_pg_id.has_value()) { return 0.0f; } + // Populate raw fields regardless of eligibility. Metrics callers include chunks that are + // currently INUSE, being GCed, or owned by a not-yet-alive PG in the "pending" backlog, so + // we must not early-return here. + snap.defrag_blks = chunk->get_defrag_nblks(); + snap.total_blks = chunk->get_total_blks(); + snap.has_pg = chunk->m_pg_id.has_value(); - // If the pg is currently destroyed or not yet alive (e.g. baseline resync), skip it; - // add_gc_task will enforce this again at submission time as a safety guard. - // FIXME: if we want avoiding GC on certain PG/CHUNK, we might added here. - if (!m_hs_home_object->is_pg_alive(chunk->m_pg_id.value())) { return 0.0f; } + if (snap.defrag_blks > 0 && snap.total_blks > 0) { + snap.ratio_pct = + (100.0f * static_cast< float >(snap.defrag_blks)) / static_cast< float >(snap.total_blks); + } - const auto total_blk_num = chunk->get_total_blks(); - const float ratio_pct = (100.0f * static_cast< float >(defrag_blk_num)) / static_cast< float >(total_blk_num); + // is_gc_candidate mirrors the original get_chunk_gc_ratio non-zero condition: + // - Only AVAILABLE chunks are candidates: INUSE means an open shard owns them, GC means + // already being processed. + // - Chunks with no pg assignment are unowned and do not need GC. + // - If the pg is destroyed or not yet alive (e.g. baseline resync), skip it; add_gc_task + // enforces this again at submission time as a safety guard. + // FIXME: if we want avoiding GC on certain PG/CHUNK, we might add here. + // Short-circuit AND on `snap.has_pg` guards the .value() call below. + const bool pg_gc_able = snap.has_pg && m_hs_home_object->is_pg_alive(chunk->m_pg_id.value()); + snap.is_gc_candidate = (chunk->m_state == ChunkState::AVAILABLE) && (snap.defrag_blks > 0) && pg_gc_able; + + // eligible adds the low-watermark cutoff — only ratios strictly above gc_thresh_low are worth + // scheduling. Matches the scanner's original `ratio_pct > gc_thresh_low` filter. + snap.eligible = snap.is_gc_candidate && (snap.ratio_pct > static_cast< float >(gc_thresh_low)); LOGDEBUGMOD(gcmgr, "gc scan chunk_id={}, use_blks={}, available_blks={}, total_blks={}, defrag_blks={}, " - "garbage_ratio_pct={}", - chunk_id, chunk->get_used_blks(), chunk->available_blks(), total_blk_num, defrag_blk_num, ratio_pct); + "garbage_ratio_pct={}, has_pg={}, is_gc_candidate={}, eligible={}", + chunk_id, chunk->get_used_blks(), chunk->available_blks(), snap.total_blks, snap.defrag_blks, + snap.ratio_pct, snap.has_pg, snap.is_gc_candidate, snap.eligible); + + return snap; +} - return ratio_pct; +float GCManager::get_chunk_gc_ratio(chunk_id_t chunk_id) { + // Preserve the original contract: return the actual ratio for chunks passing every gate + // except the low-watermark threshold; return 0 otherwise. Callers that need finer control + // (scanner, metrics accumulation) should call get_chunk_gc_snapshot directly. + const auto snap = get_chunk_gc_snapshot(chunk_id, 0 /* gc_thresh_low */); + return snap.is_gc_candidate ? snap.ratio_pct : 0.0f; } void GCManager::scan_chunks_for_gc() { @@ -273,21 +293,6 @@ void GCManager::scan_chunks_for_gc() { pdev_id); auto& actor = it->second; - // Compute remaining capacity against the true cross-scan quota. - // m_pending_normal_gc_task_count tracks all tasks currently queued or running in m_gc_executor, - // not just tasks submitted by this scan cycle. This prevents unbounded queue growth across scans. - const uint32_t already_pending = actor->get_pending_normal_task_count(); - if (already_pending >= max_task_num) { - LOGINFOMOD(gcmgr, - "pdev_id={} already has {}/{} pending normal gc tasks, skipping submission this scan cycle", - pdev_id, already_pending, max_task_num); - continue; - } - const uint32_t remaining_capacity = max_task_num - already_pending; - // Low-tier chunks (below high watermark) may consume at most half the remaining capacity so - // that high-tier chunks always get priority when quota is tight. - const uint32_t low_tier_cap = remaining_capacity / 2; - // Collect at most max_task_num chunks with the highest garbage ratios via a bounded // min-heap. K = max_task_num (fixed during the scan) rather than remaining_capacity // (which may shrink/grow as tasks queue/complete) so we always have enough candidates @@ -303,17 +308,76 @@ void GCManager::scan_chunks_for_gc() { return a.garbage_ratio_pct > b.garbage_ratio_pct; }; std::priority_queue< ChunkGCInfo, std::vector< ChunkGCInfo >, decltype(min_heap_cmp) > top_k(min_heap_cmp); + + // Snapshot accumulators for the backlog / pressure gauges. Published to actor atomics + // after the loop so metrics stay internally consistent within a scan cycle. See + // pdev_gc_actor::m_pending_* for freshness semantics (worst-case staleness = + // gc_scan_interval_sec). blk_size is pulled once per pdev to avoid repeated calls. + // + // Accumulation runs unconditionally — even when the pdev's normal-GC queue is saturated + // (see saturation check below). Losing metric refresh precisely when the queue is + // backlogged would blind operators to the pressure that is the whole point of these + // gauges; we always pay the O(N_chunks) scan cost, only the submission phase is gated. + const uint32_t blk_size = homestore::data_service().get_blk_size(); + uint64_t pending_bytes = 0; + uint64_t eligible_bytes = 0; + uint32_t eligible_chunk_count = 0; + std::array< uint32_t, 10 > pending_ratio_buckets{}; + for (const auto& chunk_id : chunks) { - const float ratio_pct = get_chunk_gc_ratio(chunk_id); - if (ratio_pct <= gc_thresh_low) { continue; } - if (top_k.size() < max_task_num) { - top_k.push({chunk_id, ratio_pct}); - } else if (ratio_pct > top_k.top().garbage_ratio_pct) { - top_k.pop(); - top_k.push({chunk_id, ratio_pct}); + const auto snap = get_chunk_gc_snapshot(chunk_id, static_cast< uint8_t >(gc_thresh_low)); + + // "Pending" set: any PG-owned chunk with garbage, regardless of state / PG liveness. + // Feeds pending_gc_bytes and the ratio bucket distribution — these describe the raw + // backlog visible to operators, not what GC will actually pick up this cycle. + if (snap.has_pg && snap.defrag_blks > 0 && snap.total_blks > 0) { + pending_bytes += static_cast< uint64_t >(snap.defrag_blks) * blk_size; + // Bucket idx via integer ceil division: (10*defrag - 1) / total. Maps ratio in + // (0, 10]% to idx 0, (10, 20]% to idx 1, ..., (90, 100]% to idx 9. Requires + // defrag_blks > 0 (guarded above) so the -1 does not underflow. + const size_t idx = std::min< size_t >( + 9, (10ULL * static_cast< uint64_t >(snap.defrag_blks) - 1ULL) / snap.total_blks); + ++pending_ratio_buckets[idx]; + } + + // "Eligible" subset: chunks the scanner would consider for a normal GC task right now. + // Fed into the min-heap for submission AND into the eligible_* gauges so operators can + // see the delta between raw backlog and what current policy actually picks up. + if (snap.eligible) { + eligible_bytes += static_cast< uint64_t >(snap.defrag_blks) * blk_size; + ++eligible_chunk_count; + + if (top_k.size() < max_task_num) { + top_k.push({chunk_id, snap.ratio_pct}); + } else if (snap.ratio_pct > top_k.top().garbage_ratio_pct) { + top_k.pop(); + top_k.push({chunk_id, snap.ratio_pct}); + } } } + // Publish the fresh snapshot for pdev_gc_metrics::on_gather to observe on the next scrape. + // This is intentionally done BEFORE the saturation short-circuit below so pending_gc_bytes, + // eligible_gc_bytes and the ratio distribution stay fresh even when the pdev is skipping + // submission — that is exactly when operators need visibility into the growing backlog. + actor->publish_scan_snapshot(pending_bytes, eligible_bytes, eligible_chunk_count, + pending_ratio_buckets); + + // Compute remaining capacity against the true cross-scan quota. + // m_pending_normal_gc_task_count tracks all tasks currently queued or running in m_gc_executor, + // not just tasks submitted by this scan cycle. This prevents unbounded queue growth across scans. + const uint32_t already_pending = actor->get_pending_normal_task_count(); + if (already_pending >= max_task_num) { + LOGINFOMOD(gcmgr, + "pdev_id={} already has {}/{} pending normal gc tasks, skipping submission this scan cycle", + pdev_id, already_pending, max_task_num); + continue; + } + const uint32_t remaining_capacity = max_task_num - already_pending; + // Low-tier chunks (below high watermark) may consume at most half the remaining capacity so + // that high-tier chunks always get priority when quota is tight. + const uint32_t low_tier_cap = remaining_capacity / 2; + // Drain the min-heap into a presized vector, writing back-to-front: the heap pops in // ascending ratio order, so placing each popped element at the current trailing index // yields a descending-by-ratio sequence directly — no separate reverse pass needed. @@ -390,6 +454,12 @@ GCManager::pdev_gc_actor::pdev_gc_actor(const homestore::superblk< GCManager::gc durable_entities_.failed_egc_task_count = gc_actor_sb->failed_egc_task_count; durable_entities_.total_reclaimed_blk_count_by_gc = gc_actor_sb->total_reclaimed_blk_count_by_gc; durable_entities_.total_reclaimed_blk_count_by_egc = gc_actor_sb->total_reclaimed_blk_count_by_egc; + + // Pre-C++20 std::atomic default-constructs to an unspecified value; zero the ratio-bucket + // atomics explicitly so metrics scrapes before the first scan return well-defined zeros. + for (auto& bucket : m_pending_ratio_buckets) { + bucket.store(0, std::memory_order_relaxed); + } } void GCManager::pdev_gc_actor::start() { diff --git a/src/lib/homestore_backend/gc_manager.hpp b/src/lib/homestore_backend/gc_manager.hpp index d8fed227..7e03153b 100644 --- a/src/lib/homestore_backend/gc_manager.hpp +++ b/src/lib/homestore_backend/gc_manager.hpp @@ -1,6 +1,9 @@ #pragma once +#include #include +#include + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #include @@ -150,6 +153,46 @@ class GCManager { gc_time_duration_s_egc, "how long a successful egc task takes by second", HistogramBucketsType(LinearUpto64Buckets)); // 17 buckets covering 0-64 seconds in 4s increments + // Backlog / pressure snapshot gauges. Values refreshed once per scan cycle by + // GCManager::scan_chunks_for_gc; worst-case staleness = gc_scan_interval_sec. + REGISTER_GAUGE(pending_gc_bytes, + "Total reclaimable garbage bytes in PG-owned chunks on this pdev"); + REGISTER_GAUGE(eligible_gc_bytes, + "Reclaimable bytes currently eligible for normal GC on this pdev"); + REGISTER_GAUGE(eligible_gc_chunk_count, + "Chunks currently eligible for normal GC on this pdev"); + REGISTER_GAUGE(pending_normal_gc_task_count, + "Normal-priority GC tasks queued or running on this pdev"); + + // Distribution of the pending backlog by garbage-ratio bucket. We register 10 + // gauges under a single Prometheus metric name (`pending_gc_chunks_ratio`), + // differentiated by a `bucket` label. The REGISTER_GAUGE macro uses a compile- + // time singleton per name and cannot express this shape (all 10 would collapse + // to one gauge index), so we call the impl directly and store the returned + // indices ourselves. Each label reads as "(lo, hi]% ratio"; the first bucket + // is (0, 10]% because a chunk contributes only if it has garbage. + // + // Descriptions embed the bucket label because sisl's JSON dump keys entries by + // description text (see MetricsGroupImpl::get_result_in_json in sisl); without + // per-bucket descriptions the 10 registrations collapse to one JSON entry + // (last-wins) and the test-log dumps become useless for verifying distribution + // shape. Prometheus text output is unaffected — it uses HELP (unchanged across + // registrations) and disambiguates series by labels. + static constexpr std::array< const char*, 10 > kRatioBucketLabels = { + "00-10", "10-20", "20-30", "30-40", "40-50", + "50-60", "60-70", "70-80", "80-90", "90-100"}; + for (size_t i = 0; i < kRatioBucketLabels.size(); ++i) { + const auto lo = i * 10; + const auto hi = (i + 1) * 10; + const auto desc = fmt::format( + "Snapshot count of pending chunks with garbage ratio in ({}, {}]% " + "(bucket={})", + lo, hi, kRatioBucketLabels[i]); + ratio_bucket_indices_[i] = m_impl_ptr->register_gauge( + "pending_gc_chunks_ratio", desc, "" /* report_name */, + sisl::metric_label{"bucket", kRatioBucketLabels[i]}); + } + register_me_to_farm(); attach_gather_cb(std::bind(&pdev_gc_metrics::on_gather, this)); } @@ -177,11 +220,29 @@ class GCManager { *this, total_reclaimed_space_by_egc, gc_actor_.durable_entities().total_reclaimed_blk_count_by_egc.load(std::memory_order_relaxed) * blk_size_); + + // Backlog / pressure snapshot gauges. Read the last-published scan accumulators. + GAUGE_UPDATE(*this, pending_gc_bytes, gc_actor_.get_pending_gc_bytes()); + GAUGE_UPDATE(*this, eligible_gc_bytes, gc_actor_.get_eligible_gc_bytes()); + GAUGE_UPDATE(*this, eligible_gc_chunk_count, gc_actor_.get_eligible_gc_chunk_count()); + GAUGE_UPDATE(*this, pending_normal_gc_task_count, gc_actor_.get_pending_normal_task_count()); + + // Bypass GAUGE_UPDATE for the same reason as bucket registration: we need to + // address 10 distinct gauge indices that share one metric name. + for (size_t i = 0; i < ratio_bucket_indices_.size(); ++i) { + m_impl_ptr->gauge_update( + ratio_bucket_indices_[i], + static_cast< int64_t >(gc_actor_.get_pending_ratio_bucket(i))); + } } private: pdev_gc_actor const& gc_actor_; uint32_t blk_size_; + // Indices returned by m_impl_ptr->register_gauge, one per ratio bucket. Populated + // during construction and consumed by on_gather (see above). Kept as a member so we + // can address each bucket by index — the compile-time-name macro cannot. + std::array< uint64_t, 10 > ratio_bucket_indices_{}; }; public: @@ -252,6 +313,36 @@ class GCManager { return m_pending_normal_gc_task_count.load(std::memory_order_relaxed); } + // Snapshot readers used by pdev_gc_metrics::on_gather. Return the last value published + // by GCManager::scan_chunks_for_gc for this pdev; 0 before the first scan completes. + uint64_t get_pending_gc_bytes() const { + return m_pending_gc_bytes.load(std::memory_order_relaxed); + } + uint64_t get_eligible_gc_bytes() const { + return m_eligible_gc_bytes.load(std::memory_order_relaxed); + } + uint32_t get_eligible_gc_chunk_count() const { + return m_eligible_gc_chunk_count.load(std::memory_order_relaxed); + } + uint32_t get_pending_ratio_bucket(size_t bucket_idx) const { + return m_pending_ratio_buckets[bucket_idx].load(std::memory_order_relaxed); + } + + // Publishes a full backlog snapshot atomically-per-field. Callers (the scanner) compute + // the totals locally over all chunks on this pdev, then hand them in via one call so the + // metrics stay internally consistent within a scan cycle. Between-gauge drift is bounded + // by one scan interval; individual scalars are aligned and therefore torn-read safe. + void publish_scan_snapshot(uint64_t pending_bytes, uint64_t eligible_bytes, + uint32_t eligible_chunks, + const std::array< uint32_t, 10 >& ratio_buckets) { + m_pending_gc_bytes.store(pending_bytes, std::memory_order_relaxed); + m_eligible_gc_bytes.store(eligible_bytes, std::memory_order_relaxed); + m_eligible_gc_chunk_count.store(eligible_chunks, std::memory_order_relaxed); + for (size_t i = 0; i < ratio_buckets.size(); ++i) { + m_pending_ratio_buckets[i].store(ratio_buckets[i], std::memory_order_relaxed); + } + } + private: void process_gc_task(chunk_id_t move_from_chunk, uint8_t priority, folly::Promise< bool > task, const uint64_t task_id); @@ -318,6 +409,16 @@ class GCManager { // Incremented in add_gc_task after a task is enqueued; decremented in on_gc_task_completed. // Used by scan_chunks_for_gc to enforce a true cross-scan quota cap. std::atomic< uint32_t > m_pending_normal_gc_task_count{0}; + + // Snapshot accumulators for the backlog / pressure gauges (see publish_scan_snapshot). + // Refreshed once per pdev iteration in GCManager::scan_chunks_for_gc; consumed by + // pdev_gc_metrics::on_gather via the get_* accessors above. Worst-case staleness = + // gc_scan_interval_sec. Relaxed ordering is sufficient because each metric is an aligned + // scalar and the values are advisory snapshots, not synchronization state. + std::atomic< uint64_t > m_pending_gc_bytes{0}; + std::atomic< uint64_t > m_eligible_gc_bytes{0}; + std::atomic< uint32_t > m_eligible_gc_chunk_count{0}; + std::array< std::atomic< uint32_t >, 10 > m_pending_ratio_buckets{}; // since we have a very small number of reserved chunks, a vector is enough // TODO:: use a map if we have a large number of reserved chunks std::vector< homestore::superblk< GCManager::gc_reserved_chunk_superblk > > m_reserved_chunks; @@ -352,6 +453,20 @@ class GCManager { // Uses floating-point arithmetic to avoid truncation for chunks with very few defrag blocks. float get_chunk_gc_ratio(chunk_id_t chunk_id); + // One-shot snapshot of chunk state relevant to GC decisions. Populated with a single + // ExtendedVChunk lookup so scan_chunks_for_gc can compute both submission decisions AND + // backlog metrics without a second hash+lock roundtrip. `gc_thresh_low` is the low-watermark + // percentage used to set the `eligible` flag; pass 0 if you only care about is_gc_candidate. + struct ChunkGCSnapshot { + uint32_t defrag_blks = 0; + uint32_t total_blks = 0; + float ratio_pct = 0.0f; // 100.0 * defrag_blks / total_blks; 0 if defrag_blks == 0 + bool has_pg = false; + bool is_gc_candidate = false; // AVAILABLE && has_pg && pg_alive && defrag_blks > 0 + bool eligible = false; // is_gc_candidate && ratio_pct > gc_thresh_low + }; + ChunkGCSnapshot get_chunk_gc_snapshot(chunk_id_t chunk_id, uint8_t gc_thresh_low); + void handle_all_recovered_gc_tasks(); void start();