diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h index 0c55316899..c0d0241bdd 100644 --- a/ddprof-lib/src/main/cpp/counters.h +++ b/ddprof-lib/src/main/cpp/counters.h @@ -210,6 +210,10 @@ * and re-emits it on a later dump while the leak candidate is still \ * live. */ \ X(REFERENCE_CHAIN_WRITE_DROPPED, "reference_chain_write_dropped") \ + /* LivenessTracker urgency boost: admissions below the table cap vs. \ + * admissions that hit the cap and fell back to the subsample ratio. */ \ + X(LIVENESS_URGENT_BOOST_ADMITS, "liveness_urgent_boost_admits") \ + X(LIVENESS_URGENT_BOOST_BACKED_OFF, "liveness_urgent_boost_backed_off") \ /* FrontierTable's own calloc/realloc-backed storage (referenceChains.cpp) - \ * outside NMT's visibility since it bypasses os::malloc, so this is the only \ * way to attribute its native RSS contribution. */ \ diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 90f6696755..904a7b6e37 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1104,6 +1104,317 @@ Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) TEST_LOG("===> Context: tid:%lu, spanId=%lu, rootSpanId=%lu", OS::threadId(), spanId, rootSpanId); } +// LivenessTracker/ReferenceChainTracker test seams. They mutate real tracker +// state, so they are only implemented in DEBUG builds; release builds get the +// no-op stubs below. +#ifdef DEBUG +#include "livenessTracker.h" +#include "referenceChains.h" +#include + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setGcGenerationsEnabled0( + JNIEnv *env, jclass unused, jboolean enabled) { + ProfiledThread::initCurrentThreadSignalSafe(); + LivenessTracker::instance()->setGcGenerationsForTest(enabled); + return JNI_TRUE; +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_seedKlassPopulationSample0( + JNIEnv *env, jclass unused, jint klassId, jint count, jlong epoch) { + ProfiledThread::initCurrentThreadSignalSafe(); + int slot; + bool created; + jweak evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int evicted_count = 0; + LivenessTracker::instance()->klassPopulationRecordForTest( + (u32)klassId, (u32)count, (u64)epoch, &slot, &created, evicted, + &evicted_count, KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + for (int i = 0; i < evicted_count; i++) { + if (evicted[i] != nullptr) { + env->DeleteWeakGlobalRef(evicted[i]); + } + } +} + +// See tidTrendRecordForTest() (livenessTracker.h) for the tid requirements. +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_seedTidTrendSample0( + JNIEnv *env, jclass unused, jint klassId, jint tid, jint count, + jlong epoch) { + ProfiledThread::initCurrentThreadSignalSafe(); + jweak evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int evicted_count = 0; + LivenessTracker::instance()->tidTrendRecordForTest( + (u32)klassId, (jint)tid, (u32)count, (u64)epoch, evicted, + &evicted_count, KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + for (int i = 0; i < evicted_count; i++) { + if (evicted[i] != nullptr) { + env->DeleteWeakGlobalRef(evicted[i]); + } + } +} + +// Makes `representative` klassId's leak-candidate representative, so a seeded +// population sample and a tagged root can drive pollWatchedTargets() without +// the real sampler or walk. The tracker owns the weak global ref created here. +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setKlassPopulationRepresentativeForTest0( + JNIEnv *env, jclass unused, jint klassId, jobject representative) { + ProfiledThread::initCurrentThreadSignalSafe(); + jweak rep = env->NewWeakGlobalRef(representative); + LivenessTracker::instance()->klassPopulationSetRepresentativeForTest( + env, (u32)klassId, rep); +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_resetKlassPopulationForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + LivenessTracker::instance()->klassPopulationResetForTest(); +} + +extern "C" DLLEXPORT jintArray JNICALL +Java_com_datadoghq_profiler_JavaProfiler_selectLeakCandidateKlassIds0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + KlassCandidate candidates[5]; + int n = LivenessTracker::instance()->selectLeakCandidates(candidates, 5); + jintArray result = env->NewIntArray(n); + if (result == nullptr || n == 0) { + return result; + } + jint ids[5]; + for (int i = 0; i < n; i++) { + ids[i] = (jint)candidates[i].klass_id; + } + env->SetIntArrayRegion(result, 0, n, ids); + return result; +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_tagAsReferenceChainRoot0( + JNIEnv *env, jclass unused, jobject target) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr) { + return 0; + } + return ReferenceChainTracker::instance()->tagAsRootForTest(jvmti, env, + target); +} + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_runReferenceChainPass0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr) { + return JNI_FALSE; + } + return ReferenceChainTracker::instance()->runPassSerialized(jvmti, env); +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_pollReferenceChainTargets0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr) { + return; + } + ReferenceChainTracker::instance()->finishLoopIterationSerialized(jvmti, env); +} + +extern "C" DLLEXPORT jint JNICALL +Java_com_datadoghq_profiler_JavaProfiler_drainReferenceChainEventCount0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + std::vector events; + ReferenceChainTracker::instance()->drainPendingChainEvents(&events); + return (jint)events.size(); +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_resetReferenceChainSearchForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr) { + return; + } + ReferenceChainTracker::instance()->resetSearchStateForTest(jvmti, env); +} + +// Reads target's existing tag without tagging it. Return values are described +// at pendingExpandPositionForTest(). +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingPositionForTest0( + JNIEnv *env, jclass unused, jobject target) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr || target == nullptr) { + return -2; + } + jlong tag = 0; + jvmtiError err = jvmti->GetTag(target, &tag); + if (err != JVMTI_ERROR_NONE) { + return -2; + } + return (jlong)ReferenceChainTracker::instance()->pendingExpandPositionForTest( + tag); +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingSizeForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + return (jlong)ReferenceChainTracker::instance()->pendingExpandSizeForTest(); +} + +// Seeds one heap-floor sample for secondsToOOM() without waiting for a GC. +// Timestamps are only compared with each other, so any increasing sequence works. +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_heapFloorRecordForTest0( + JNIEnv *env, jclass unused, jlong usedBytes, jlong timestampNs) { + ProfiledThread::initCurrentThreadSignalSafe(); + LivenessTracker::instance()->heapFloorRecordForTest((u64)usedBytes, + (u64)timestampNs); +} + +// Overrides the max heap size secondsToOOM() projects against. +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setMaxHeapBytesForTest0( + JNIEnv *env, jclass unused, jlong maxHeapBytes) { + ProfiledThread::initCurrentThreadSignalSafe(); + LivenessTracker::instance()->setMaxHeapBytesForTest((jlong)maxHeapBytes); +} + +// Stops real GCs from recording heap-floor samples while a test seeds them. +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setHeapFloorRecordingForTest0( + JNIEnv *env, jclass unused, jboolean enabled) { + ProfiledThread::initCurrentThreadSignalSafe(); + LivenessTracker::instance()->setHeapFloorRecordingForTest(enabled == JNI_TRUE); +} + +// Evaluates the real scheduling gate without executing a BFS pass. This is not read-only: under +// the engine lock it may account completed-search pain and restart/reset terminal search state. +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_shouldRunPassForTest0(JNIEnv *env, + jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + return ReferenceChainTracker::instance()->shouldRunPassForTest( + OS::nanotime()) + ? JNI_TRUE + : JNI_FALSE; +} + +// Lets a test wait for a pass that started after it created an object. +extern "C" DLLEXPORT jint JNICALL +Java_com_datadoghq_profiler_JavaProfiler_referenceChainPassesRunForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + return (jint)ReferenceChainTracker::instance()->passesRun(); +} + +#else // !DEBUG + +// Release builds: no-op stubs so the Java natives still link. +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setGcGenerationsEnabled0( + JNIEnv *env, jclass unused, jboolean enabled) { + return JNI_FALSE; +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_seedKlassPopulationSample0( + JNIEnv *env, jclass unused, jint klassId, jint count, jlong epoch) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_seedTidTrendSample0( + JNIEnv *env, jclass unused, jint klassId, jint tid, jint count, + jlong epoch) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setKlassPopulationRepresentativeForTest0( + JNIEnv *env, jclass unused, jint klassId, jobject representative) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_resetKlassPopulationForTest0( + JNIEnv *env, jclass unused) {} + +extern "C" DLLEXPORT jintArray JNICALL +Java_com_datadoghq_profiler_JavaProfiler_selectLeakCandidateKlassIds0( + JNIEnv *env, jclass unused) { + return env->NewIntArray(0); +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_tagAsReferenceChainRoot0( + JNIEnv *env, jclass unused, jobject target) { + return 0; +} + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_runReferenceChainPass0( + JNIEnv *env, jclass unused) { + return JNI_FALSE; +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_pollReferenceChainTargets0( + JNIEnv *env, jclass unused) {} + +extern "C" DLLEXPORT jint JNICALL +Java_com_datadoghq_profiler_JavaProfiler_drainReferenceChainEventCount0( + JNIEnv *env, jclass unused) { + return 0; +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_resetReferenceChainSearchForTest0( + JNIEnv *env, jclass unused) {} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingPositionForTest0( + JNIEnv *env, jclass unused, jobject target) { + return -2; +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingSizeForTest0( + JNIEnv *env, jclass unused) { + return 0; +} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_heapFloorRecordForTest0( + JNIEnv *env, jclass unused, jlong usedBytes, jlong timestampNs) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setMaxHeapBytesForTest0( + JNIEnv *env, jclass unused, jlong maxHeapBytes) {} + +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_setHeapFloorRecordingForTest0( + JNIEnv *env, jclass unused, jboolean enabled) {} + +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_shouldRunPassForTest0(JNIEnv *env, + jclass unused) { + return JNI_FALSE; +} + +extern "C" DLLEXPORT jint JNICALL +Java_com_datadoghq_profiler_JavaProfiler_referenceChainPassesRunForTest0( + JNIEnv *env, jclass unused) { + return 0; +} + +#endif // DEBUG + // ---- Test-only reads of the current thread's OTEP record ----------------------------------- // Each reads the current carrier's record directly via ProfiledThread::current(), with no // detach/attach (diagnostic-only, not on any signal-handler or hot write path). diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp index b4a05ef97b..a3a1a6afe1 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ b/ddprof-lib/src/main/cpp/livenessTracker.cpp @@ -13,6 +13,7 @@ #include "common.h" #include "context.h" #include "context_api.h" +#include "counters.h" #include "hotspot/vmStructs.h" #include "hotspot/vmStructs.inline.h" #include "incbin.h" @@ -63,12 +64,12 @@ constexpr u32 RESOLVE_BUDGET_PER_SWEEP = 256; // discipline for physical slot `i` without this shared loop needing to know // which one applies. // -// DESPITE THE NAME, this is not thirds statistics: the design doc's original -// "mean of earliest third vs mean of recent third" comparison was replaced by -// a full-window least-squares linear regression (see ringThirdsStats below), -// which uses all samples and is far more robust for oscillating-but-growing -// trends. The field names survive as the regression values consumers treat -// as the window's "earliest"/"recent" levels: +// Not thirds statistics: an earlier "mean of earliest third vs mean of +// recent third" comparison was replaced by a full-window least-squares linear +// regression (see ringWindowStats below), which uses all samples and is far +// more robust for oscillating-but-growing trends. The field names survive as +// the regression values consumers treat as the window's "earliest"/"recent" +// levels: // earliest_mean - regression value at the window's OLDEST sample (x = 0); // recent_mean - regression value at the window's NEWEST sample (x = fill-1); // earliest_min - true minimum over the FULL window; @@ -77,7 +78,7 @@ constexpr u32 RESOLVE_BUDGET_PER_SWEEP = 256; // (a regression slope over the window's span) and earliest_min/recent_min as // floor checks. Renaming the fields would touch every consumer for no // behavioral change, so the mapping is documented here instead. -struct RingThirdsStats { +struct RingWindowStats { double earliest_mean; // regression value at the window's oldest sample double recent_mean; // regression value at the window's newest sample double earliest_min; // true min over the full window @@ -85,8 +86,8 @@ struct RingThirdsStats { }; template -bool ringThirdsStats(int head, int fill, int ring_size, int min_fill, - Reader read, RingThirdsStats *out) { +bool ringWindowStats(int head, int fill, int ring_size, int min_fill, + Reader read, RingWindowStats *out) { if (fill < min_fill) { return false; } @@ -134,7 +135,7 @@ bool ringThirdsStats(int head, int fill, int ring_size, int min_fill, // Recent-half corroboration for a usage ring (see secondsToOOM()'s own // comment): a rising full-window trend whose most recent half is flat is a // plateaued step change, not ongoing growth. The recent half's own -// regression (see ringThirdsStats) must show a strictly positive delta for +// regression (see ringWindowStats) must show a strictly positive delta for // the full-window trend to stand. A too-sparse recent half (below min_fill) // REJECTS the projection rather than letting it through: false means // "reject". Deliberately stricter than the single-ring version's @@ -146,8 +147,8 @@ template bool corroborateRecentHalf(u8 head, u8 fill, int ring_size, int min_fill, Reader read) { int half_fill = fill / 2; - RingThirdsStats recent_half_stats; - bool have_half = ringThirdsStats(head, half_fill, ring_size, min_fill, read, + RingWindowStats recent_half_stats; + bool have_half = ringWindowStats(head, half_fill, ring_size, min_fill, read, &recent_half_stats); double half_delta = have_half ? recent_half_stats.recent_mean - recent_half_stats.earliest_mean @@ -157,7 +158,8 @@ bool corroborateRecentHalf(u8 head, u8 fill, int ring_size, int min_fill, } // namespace -void LivenessTracker::cleanup_table(bool forced, bool allow_resolve) { +void LivenessTracker::cleanup_table(bool forced, bool allow_resolve, + bool account_epoch) { u64 current = load(_last_gc_epoch); u64 target_gc_epoch = load(_gc_epoch); TEST_LOG_SUMMARY("LivenessTracker::cleanup_table forced=%d gc_generations=%d current_epoch=%llu " @@ -197,16 +199,13 @@ void LivenessTracker::cleanup_table(bool forced, bool allow_resolve) { // caller whose target is strictly newer claims; a stale snapshot skips // epoch accounting entirely (its survivors are still swept, their ages // simply do not move for this no-op epoch). - bool is_epoch_owner = target_gc_epoch > claimed && + // account_epoch=false only reaps: claiming the epoch here would make the + // background sweep skip its population fold for it. + bool is_epoch_owner = account_epoch && target_gc_epoch > claimed && __atomic_compare_exchange_n(&_last_gc_epoch, &claimed, target_gc_epoch, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); - // On a lost CAS race `claimed` holds the actual current epoch (>= our - // stale target), so the raw diff is <= 0 for every non-owner; clamp so a - // survivor's unsigned age can never wrap. - int epoch_diff = (int)(target_gc_epoch - claimed); - if (epoch_diff < 0) { - epoch_diff = 0; - } + // Only the epoch owner ages survivors; otherwise ages could wrap (lost + // CAS race) or advance twice (account_epoch=false). // Detect a class-map reset the same way // ReferenceChainTracker::resolveLoadedClasses() does (referenceChains.cpp) @@ -266,7 +265,15 @@ void LivenessTracker::cleanup_table(bool forced, bool allow_resolve) { _table[i].cached_klass_id = 0; __atomic_store_n(&_table[i].ready, 0, __ATOMIC_RELEASE); } - _table[target].age += epoch_diff; + // Don't age an entry for GCs that happened before it was admitted. + if (is_epoch_owner) { + u64 base = _table[target].admission_epoch > claimed + ? _table[target].admission_epoch + : claimed; + if (target_gc_epoch > base) { + _table[target].age += (jlong)(target_gc_epoch - base); + } + } if (_gc_generations.load(std::memory_order_relaxed) && is_epoch_owner) { // Per-klass population tracking (design doc's Open Question 3) - @@ -346,20 +353,20 @@ void LivenessTracker::cleanup_table(bool forced, bool allow_resolve) { TEST_LOG_SUMMARY("LivenessTracker::cleanup_table survivors=%u klass_count_scratch_size=%d", newsz, _klass_count_scratch_size); - if (_gc_generations.load(std::memory_order_relaxed) && is_epoch_owner) { - // Runs even when _klass_count_scratch is empty: foldKlassCountsLocked() - // records zero population samples for klasses whose every tracked - // instance died this epoch (they never appear in the scratch, and - // without a zero sample a dead population would stay a leak candidate - // until its entry is evicted). - foldKlassCountsLocked(env, target_gc_epoch, allow_resolve); - } end = OS::nanotime(); Log::debug("Liveness tracker cleanup took %.2fms (%.2fus/element)", 1.0f * (end - start) / 1000 / 1000, 1.0f * (end - start) / 1000 / sz); } + if (_gc_generations.load(std::memory_order_relaxed) && is_epoch_owner) { + // Outside the `sz > 0` block: a sweep that finds the table already empty + // still owns this epoch. foldKlassCountsLocked() records zero population + // samples for klasses whose every tracked instance died (they never + // appear in the scratch, and without a zero sample a dead population + // would stay a leak candidate until its entry is evicted). + foldKlassCountsLocked(env, target_gc_epoch, allow_resolve); + } _table_lock.unlock(); } } @@ -1279,8 +1286,8 @@ void LivenessTracker::foldKlassCountsLocked(JNIEnv *env, u64 epoch, } bool LivenessTracker::hasQualifyingGrowth(const KlassPopulationEntry &entry) const { - RingThirdsStats stats; - if (!ringThirdsStats( + RingWindowStats stats; + if (!ringWindowStats( entry.ring_head, entry.ring_fill, KLASS_POPULATION_RING_SIZE, KLASS_POPULATION_MIN_FILL_FOR_TREND, [&entry](int i) { return (double)entry.count_ring[i]; }, &stats)) { @@ -1315,8 +1322,8 @@ bool LivenessTracker::hasQualifyingGrowth(const KlassPopulationEntry &entry) con bool LivenessTracker::hasQualifyingTidGrowth( const KlassPopulationEntry::TidTrend &trend) const { - RingThirdsStats stats; - if (!ringThirdsStats( + RingWindowStats stats; + if (!ringWindowStats( trend.ring_head, trend.ring_fill, KlassPopulationEntry::TID_TREND_RING_SIZE, TID_TREND_MIN_FILL_FOR_TREND, @@ -1507,8 +1514,8 @@ bool LivenessTracker::heapFloorRising() const { // loadAcquire() here is what makes the payload writes below visible. u8 fill = loadAcquire(_heap_floor_ring_fill); u8 head = loadAcquire(_heap_floor_ring_head); - RingThirdsStats stats; - if (!ringThirdsStats( + RingWindowStats stats; + if (!ringWindowStats( head, fill, KLASS_POPULATION_RING_SIZE, KLASS_POPULATION_MIN_FILL_FOR_TREND, [this](int i) { return (double)load(_heap_floor_ring[i]); }, @@ -1580,8 +1587,8 @@ double LivenessTracker::secondsToOOM() const { (int)fill, KLASS_POPULATION_MIN_FILL_FOR_TREND); return -1; } - RingThirdsStats time_stats; - if (!ringThirdsStats( + RingWindowStats time_stats; + if (!ringWindowStats( head, fill, KLASS_POPULATION_RING_SIZE, KLASS_POPULATION_MIN_FILL_FOR_TREND, [this](int i) { return (double)load(_heap_floor_time_ring[i]); }, @@ -1599,45 +1606,53 @@ double LivenessTracker::secondsToOOM() const { const char *best_source = "none"; double best_recent_mean = 0; - RingThirdsStats heap_bytes; + RingWindowStats heap_bytes; if (max_heap > 0 && - ringThirdsStats(head, fill, KLASS_POPULATION_RING_SIZE, + ringWindowStats(head, fill, KLASS_POPULATION_RING_SIZE, KLASS_POPULATION_MIN_FILL_FOR_TREND, [this](int i) { return (double)load(_heap_floor_ring[i]); }, &heap_bytes) && corroborateRecentHalf(head, fill, KLASS_POPULATION_RING_SIZE, HEAP_FLOOR_RECENT_HALF_MIN_FILL, [this](int i) { return (double)load(_heap_floor_ring[i]); })) { - double remaining = (double)max_heap - heap_bytes.recent_mean; - double secs = remaining <= 0 - ? 0 - : (remaining * time_delta_ns) / - (heap_bytes.recent_mean - heap_bytes.earliest_mean) / 1e9; - if (best_seconds < 0 || secs < best_seconds) { - best_seconds = secs; - best_source = "heap"; - best_recent_mean = heap_bytes.recent_mean; + // A dip-then-recover window passes corroborateRecentHalf() but can have a + // zero or negative full-window delta; no projection then. + double heap_delta = heap_bytes.recent_mean - heap_bytes.earliest_mean; + if (heap_delta > 0) { + double remaining = (double)max_heap - heap_bytes.recent_mean; + double secs = remaining <= 0 + ? 0 + : (remaining * time_delta_ns) / heap_delta / 1e9; + if (best_seconds < 0 || secs < best_seconds) { + best_seconds = secs; + best_source = "heap"; + best_recent_mean = heap_bytes.recent_mean; + } } } - RingThirdsStats container_bytes; + RingWindowStats container_bytes; if (container_limit > 0 && - ringThirdsStats(head, fill, KLASS_POPULATION_RING_SIZE, + ringWindowStats(head, fill, KLASS_POPULATION_RING_SIZE, KLASS_POPULATION_MIN_FILL_FOR_TREND, [this](int i) { return (double)load(_container_mem_ring[i]); }, &container_bytes) && corroborateRecentHalf(head, fill, KLASS_POPULATION_RING_SIZE, HEAP_FLOOR_RECENT_HALF_MIN_FILL, [this](int i) { return (double)load(_container_mem_ring[i]); })) { - double remaining = (double)container_limit - container_bytes.recent_mean; - double secs = remaining <= 0 - ? 0 - : (remaining * time_delta_ns) / - (container_bytes.recent_mean - container_bytes.earliest_mean) / 1e9; - if (best_seconds < 0 || secs < best_seconds) { - best_seconds = secs; - best_source = "container"; - best_recent_mean = container_bytes.recent_mean; + // Same guard as for the heap. + double container_delta = + container_bytes.recent_mean - container_bytes.earliest_mean; + if (container_delta > 0) { + double remaining = (double)container_limit - container_bytes.recent_mean; + double secs = remaining <= 0 + ? 0 + : (remaining * time_delta_ns) / container_delta / 1e9; + if (best_seconds < 0 || secs < best_seconds) { + best_seconds = secs; + best_source = "container"; + best_recent_mean = container_bytes.recent_mean; + } } } @@ -1993,6 +2008,11 @@ Error LivenessTracker::initialize_table(JNIEnv *jni, int sampling_interval) { return Error::OK; } +void LivenessTracker::resetSession() { + _enabled = false; + _gc_generations.store(false, std::memory_order_relaxed); +} + Error LivenessTracker::start(Arguments &args) { Error err = initialize(args); if (err) { @@ -2195,7 +2215,18 @@ static ThreadLocal skipped; // per thread and probabilistic by design). bool LivenessTracker::admitForTracking(jint tid) { if (__atomic_load_n(&_urgent_tracking, __ATOMIC_ACQUIRE)) { - return true; + // Volume backstop for the urgency boost: with a full table every admission + // forces a sweep on the allocation callback, so fall back to the + // watched-tid boost and the subsample ratio. Heuristic only; a stale + // relaxed read while cleanup_table() runs is acceptable. + bool back_off = _table_max_cap > 0 && + __atomic_load_n(&_table_size, __ATOMIC_RELAXED) >= _table_max_cap; + if (back_off) { + Counters::increment(LIVENESS_URGENT_BOOST_BACKED_OFF); + } else { + Counters::increment(LIVENESS_URGENT_BOOST_ADMITS); + return true; + } } // Count+array two-phase publish (noteSelectedCandidates() writes the // slots before release-storing the count): the acquire load pairs with @@ -2359,6 +2390,7 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, _table[idx].skipped = skipped.get(); skipped.set(0); _table[idx].age = 0; + _table[idx].admission_epoch = load(_gc_epoch); _table[idx].call_trace_id = call_trace_id; _table[idx].leak_tag = 0; _table[idx].ctx = ContextApi::snapshot(); @@ -2379,7 +2411,8 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, // space. allow_resolve=false: this runs synchronously on the // allocation-sampling callback stack (see cleanup_table()'s own header // comment for why resolveKlassId() is unsafe here). - cleanup_table(true, false); + // Reap only; the population fold is too heavy for the allocation callback. + cleanup_table(true, false, false); if (_table_cap < _table_max_cap) { diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index fb5561c17d..a10fb017cc 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -30,6 +30,10 @@ typedef struct TrackingEntry { jint tid; jlong time; jlong age; + // _gc_epoch at admission. An entry admitted after an account_epoch=false + // sweep can be in the table before the owner claims that epoch; aging + // starts from here so it is not credited with a GC it never survived. + u64 admission_epoch; jlong leak_tag; // 0 = untagged; otherwise a tag from the leak tag pool Context ctx; // Set by cleanup_table()'s survivor loop via resolveKlassId() when @@ -103,7 +107,7 @@ typedef struct KlassPopulationEntry { // often as a real leak does. u8 consecutive_positive; // Slope (regression value at the ring's newest sample minus the value at - // its oldest sample - see ringThirdsStats, livenessTracker.cpp) as of the + // its oldest sample - see ringWindowStats, livenessTracker.cpp) as of the // last push, computed and cached by hasQualifyingGrowth() alongside // consecutive_positive above - selectLeakCandidates() reads this directly // for ranking instead of re-scanning the ring: the ring only changes on @@ -601,8 +605,10 @@ class alignas(alignof(SpinLock)) LivenessTracker { // thread (referenceChains.cpp), not the allocation hot path, so the same // upcalls flush_table() already makes safely are just as safe there - see // that method's own comment for why a third caller needs both bypassing - // the early-exit *and* resolution. - void cleanup_table(bool force = false, bool allow_resolve = true); + // the early-exit *and* resolution. account_epoch=false only reaps: no epoch + // claim, no survivor aging, no population fold. + void cleanup_table(bool force = false, bool allow_resolve = true, + bool account_epoch = true); void flush_table(std::set *tracked_thread_ids); @@ -755,10 +761,10 @@ class alignas(alignof(SpinLock)) LivenessTracker { // The sustained-trend gate (this class's own header comment above, // "Sustained-trend gate") - both-required growth-magnitude and floor-rise // tests, design doc's original "mean of thirds" choice since replaced by - // full-window least-squares regression (see ringThirdsStats, + // full-window least-squares regression (see ringWindowStats, // livenessTracker.cpp - cheap, allocation-free, one pass over the // ring, no sorting or extra storage). A single scan - // (ringThirdsStats(), livenessTracker.cpp) both derives the pass/fail + // (ringWindowStats(), livenessTracker.cpp) both derives the pass/fail // result below AND updates entry.cached_slope (regression end value minus // start value) for selectLeakCandidates()'s ranking, rather than // that method re-scanning the same unchanged ring a moment later. Returns @@ -830,6 +836,9 @@ class alignas(alignof(SpinLock)) LivenessTracker { _last_class_map_generation(0), _leak_tag_free_count(LEAK_TAG_POOL_SIZE) {} + // Reset recording-scoped gates at every accepted profiler start. Persistent tracking and + // population state intentionally survives recording boundaries. + void resetSession(); Error start(Arguments &args); void stop(); void track(JNIEnv *env, AllocEvent &event, jint tid, jobject object, u64 call_trace_id); @@ -1238,7 +1247,10 @@ class alignas(alignof(SpinLock)) LivenessTracker { // above. Out of gtest's reach in one respect: gtest call sites have no // live tracked instances to tag anyway (they exercise the qualification // gate only), so a distinct gtest-chosen tid is fine there. - void tidTrendRecordForTest(u32 klass_id, jint tid, u32 count, u64 epoch) { + void tidTrendRecordForTest(u32 klass_id, jint tid, u32 count, u64 epoch, + jweak *out_evicted = nullptr, + int *out_evicted_count = nullptr, + int max_evicted = 0) { _table_lock.lock(); klass_id = resolveTestKlassAliasLocked(klass_id); int slot = -1; @@ -1251,7 +1263,9 @@ class alignas(alignof(SpinLock)) LivenessTracker { if (slot < 0) { int out_slot; bool created; - recordKlassPopulationSampleLocked(klass_id, 0, 0, &out_slot, &created); + recordKlassPopulationSampleLocked(klass_id, 0, 0, &out_slot, &created, + out_evicted, out_evicted_count, + max_evicted); slot = out_slot; } KlassPopulationEntry &entry = _klass_population[slot]; diff --git a/ddprof-lib/src/main/cpp/objectSampler.cpp b/ddprof-lib/src/main/cpp/objectSampler.cpp index 4232581200..38a879484b 100644 --- a/ddprof-lib/src/main/cpp/objectSampler.cpp +++ b/ddprof-lib/src/main/cpp/objectSampler.cpp @@ -153,6 +153,12 @@ void ObjectSampler::recordAllocation(jvmtiEnv *jvmti, JNIEnv *jni, } } +void ObjectSampler::resetSession() { + _record_allocations = false; + _record_liveness = false; + _gc_generations = false; +} + Error ObjectSampler::check(Arguments &args) { if (!VM::canSampleObjects()) { return Error("Allocation Sampling is not supported on this JVM"); @@ -182,11 +188,16 @@ Error ObjectSampler::start(Arguments &args) { return error; } if (_interval > 0) { - if (_record_liveness || _gc_generations) { - error = LivenessTracker::instance()->start(args); - if (error) { - return error; - } + // LivenessTracker is started unconditionally (below) so it picks up this + // recording's flags instead of keeping the previous recording's. + // It must finish before callbacks are enabled: initialize_table() + // publishes _table_cap before _table is allocated. + _alloc_event_count = 0; + error = LivenessTracker::instance()->start(args); + if (error) { + resetSession(); + LivenessTracker::instance()->resetSession(); + return error; } jvmtiEnv *jvmti = VM::jvmti(); @@ -194,13 +205,10 @@ Error ObjectSampler::start(Arguments &args) { // used by one JVMTI environment. Therefore, we can rely on the fact that if // this agent gets hold of the sample it will be its exclusive owner. jvmti->SetHeapSamplingInterval(_interval); + __atomic_store_n(&_last_config_update_ts, OS::nanotime(), __ATOMIC_RELEASE); jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_SAMPLED_OBJECT_ALLOC, NULL); __atomic_store_n(&_active, true, __ATOMIC_RELEASE); - __atomic_store_n(&_last_config_update_ts, OS::nanotime(), __ATOMIC_RELEASE); - // need to reset the running sum in order for 'updateConfiguration' to be - // able to generate proper diffs - _alloc_event_count = 0; } return Error::OK; @@ -212,9 +220,8 @@ void ObjectSampler::stop() { jvmti->SetEventNotificationMode(JVMTI_DISABLE, JVMTI_EVENT_SAMPLED_OBJECT_ALLOC, NULL); - if (_record_liveness || _gc_generations) { - LivenessTracker::instance()->stop(); - } + // No-op when the tracker is not enabled. + LivenessTracker::instance()->stop(); } Error ObjectSampler::updateConfiguration(u64 events, double time_coefficient) { diff --git a/ddprof-lib/src/main/cpp/objectSampler.h b/ddprof-lib/src/main/cpp/objectSampler.h index 88718087cb..ff183c4cee 100644 --- a/ddprof-lib/src/main/cpp/objectSampler.h +++ b/ddprof-lib/src/main/cpp/objectSampler.h @@ -67,6 +67,8 @@ class ObjectSampler : public Engine { public: static ObjectSampler *const instance() { return _instance; } + // Reset per-recording metadata gates even when this optional engine will not start. + void resetSession(); Error check(Arguments &args); Error start(Arguments &args); void stop(); diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 119e08e74f..778cc5bbdd 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1504,6 +1504,11 @@ Error Profiler::start(Arguments &args, bool reset) { return Error("No profiling events specified"); } + // Recording-scoped liveness and allocation metadata must not survive when the new recording + // does not activate the optional allocation engine. + LivenessTracker::instance()->resetSession(); + ObjectSampler::instance()->resetSession(); + // Commit _features before the reset block so any signal-handler code that // reads _features.* observes the correct enabled state once profiling // engines start. @@ -2072,6 +2077,9 @@ Error Profiler::dump(const char *path, const int length) { err = _jfr.dump(path, length); __atomic_add_fetch(&_epoch, 1, __ATOMIC_SEQ_CST); }); + if (err) { + TEST_LOG("Profiler::dump _jfr.dump failed: %s", err.message()); + } _thread_info.clearAll(thread_ids); _thread_info.reportCounters(); diff --git a/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp b/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp index 6da01861a2..f46158e822 100644 --- a/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainAnchors.cpp @@ -81,9 +81,12 @@ void ReferenceChainTracker::descendFromAnchor( u32 anchor_depth, jlong anchor_descend_class_tag, int budget, int *edges_admitted, bool *truncated, bool *frontier_cap_hit, u64 *safepoint_ns) { + jlong resolved_chain_tags[MAX_RESOLVED_CHAINS]; ReferenceChainPassContext ctx; ctx.tracker = this; ctx.frontier = _frontier; + ctx.resolved_chain_tags = resolved_chain_tags; + ctx.resolved_chain_tag_count = snapshotResolvedChainTags(resolved_chain_tags); // Bound admission to DESCENT_HOPS below the anchor, still subject to the global hop cap. int descent_cap = (int)anchor_depth + DESCENT_HOPS; ctx.hop_cap = descent_cap < _hop_cap ? descent_cap : _hop_cap; diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index b2a8052187..de60c15403 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -149,6 +149,9 @@ void ReferenceChainTracker::pollWatchedTargets(jvmtiEnv *jvmti, JNIEnv *jni) { return; } + // Don't rebuild chains against cache entries the heap callback asked to drop. + drainPendingChainInvalidations(); + // Stamp every entry this poll refreshes with the current search generation. const u64 current_search_ns = load(_search_start_ns); @@ -496,6 +499,77 @@ void ReferenceChainTracker::invalidateResolvedChain(jlong source_tag) { } } +int ReferenceChainTracker::snapshotResolvedChainTags(jlong *out) { + if (out == nullptr) { + return 0; + } + ExclusiveLockGuard guard(&_resolved_chains_lock); + int count = 0; + for (const auto &entry : _resolved_chains) { + assert(count < MAX_RESOLVED_CHAINS); + out[count++] = entry.first; + } + return count; +} + +void ReferenceChainTracker::deferResolvedChainInvalidation( + jlong source_tag, const jlong *resolved_chain_tags, + int resolved_chain_tag_count) { + bool was_cached = false; + for (int i = 0; i < resolved_chain_tag_count; i++) { + if (resolved_chain_tags[i] == source_tag) { + was_cached = true; + break; + } + } + if (!was_cached) { + return; + } + + _pending_chain_invalidations_lock.lock(); + for (jlong tag : _pending_chain_invalidations) { + if (tag == source_tag) { + _pending_chain_invalidations_lock.unlock(); + return; + } + } + // Cache insertion is serialized with traversals on the engine thread and dumps can only remove + // keys, so one pre-traversal snapshot bounds distinct eligible tags to the cache cap. + assert((int)_pending_chain_invalidations.size() < + MAX_PENDING_CHAIN_INVALIDATIONS); + if ((int)_pending_chain_invalidations.size() < + MAX_PENDING_CHAIN_INVALIDATIONS) { + _pending_chain_invalidations.push_back(source_tag); + } + _pending_chain_invalidations_lock.unlock(); +} + +void ReferenceChainTracker::drainPendingChainInvalidations() { + // Serialize drainers and scratch-buffer use with the cache. The producer lock is held only for + // the O(1) handoff, never for unordered_map erase or node destruction. + ExclusiveLockGuard guard(&_resolved_chains_lock); + _pending_chain_invalidations_lock.lock(); + assert(_draining_chain_invalidations.empty()); + _pending_chain_invalidations.swap(_draining_chain_invalidations); + _pending_chain_invalidations_lock.unlock(); + + for (jlong tag : _draining_chain_invalidations) { + if (_resolved_chains.erase(tag) > 0) { + TEST_LOG("ReferenceChainTracker::drainPendingChainInvalidations source_tag=%lld", + (long long)tag); + } + } + _draining_chain_invalidations.clear(); +} + +void ReferenceChainTracker::clearPendingChainInvalidations() { + ExclusiveLockGuard cache_guard(&_resolved_chains_lock); + _pending_chain_invalidations_lock.lock(); + _pending_chain_invalidations.clear(); + _draining_chain_invalidations.clear(); + _pending_chain_invalidations_lock.unlock(); +} + // Builds and caches chain events for every auto-marked discovered instance recorded against a slot // holding klass_id (see the auto-mark block in heapReferenceCallback() for how instances get // recorded). @@ -595,9 +669,10 @@ for (int s = 0; s < _candidate_count; s++) { } } -void ReferenceChainTracker::recordDiscoveredInstance(u32 klass_id, - jlong frontier_tag, - bool leak_correlated) { +void ReferenceChainTracker::recordDiscoveredInstance( + u32 klass_id, jlong frontier_tag, bool leak_correlated, + CacheInvalidationMode invalidation_mode, + const jlong *resolved_chain_tags, int resolved_chain_tag_count) { // See the declaration's own comment (referenceChains.h) for the noise-eviction rationale. for (int s = 0; s < _candidate_count; s++) { if (_candidate_klass_ids[s] != klass_id) { @@ -624,7 +699,12 @@ void ReferenceChainTracker::recordDiscoveredInstance(u32 klass_id, !_frontier->lookup(victim, &victim_entry) || victim_entry.leak_tag == 0) { _candidate_discovered_tags[s][d] = frontier_tag; - invalidateResolvedChain(victim); + if (invalidation_mode == CacheInvalidationMode::DEFERRED) { + deferResolvedChainInvalidation(victim, resolved_chain_tags, + resolved_chain_tag_count); + } else { + invalidateResolvedChain(victim); + } TEST_LOG("ReferenceChainTracker::recordDiscoveredInstance evicted " "noise slot=%d idx=%d victim_tag=%lld for leak tag=%lld", s, d, (long long)victim, (long long)frontier_tag); @@ -662,7 +742,8 @@ bool ReferenceChainTracker::correlateAdmittedLeakTag(jlong frontier_tag, "frontier_tag=%lld leak_tag=%lld depth=%u parent_tag=%lld", (long long)frontier_tag, (long long)leak_tag, entry.depth, (long long)entry.parent_tag); - recordDiscoveredInstance(klass_id, frontier_tag, true); + recordDiscoveredInstance(klass_id, frontier_tag, true, + CacheInvalidationMode::SYNCHRONOUS, nullptr, 0); return true; } @@ -671,6 +752,8 @@ void ReferenceChainTracker::drainPendingChainEvents( if (out == nullptr) { return; } + // Apply deferred evictions first so the dump doesn't export an already-evicted chain. + drainPendingChainInvalidations(); // Snapshot-and-keep, not a drain: every cached chain is copied out (and re-stamped so it lands in // the dumping chunk's window) while the cache itself is left intact, so the same live sample's // chain re-emits into every chunk it survives into (see _resolved_chains' comment). diff --git a/ddprof-lib/src/main/cpp/referenceChainInternal.h b/ddprof-lib/src/main/cpp/referenceChainInternal.h index 2d2248230c..f4c5ad98e5 100644 --- a/ddprof-lib/src/main/cpp/referenceChainInternal.h +++ b/ddprof-lib/src/main/cpp/referenceChainInternal.h @@ -29,6 +29,12 @@ struct ReferenceChainPassContext { // exhaustion, as opposed to edges_admitted reaching budget. bool frontier_cap_hit; + // Immutable snapshot of the resolved-chain source tags that were cached before this synchronous + // traversal started. Callback code uses this fixed caller-owned array to decide whether a shape + // change needs invalidation without taking the resolved-chain cache lock. + const jlong *resolved_chain_tags = nullptr; + int resolved_chain_tag_count = 0; + // ARRAY-HOLDER BATCHING: when non-null, expandFrontier() is driving a one-hop expansion of a // batch of boundary objects passed to a single FollowReferences(initial_object=holder_array) // call. diff --git a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp index d96cc01450..8d6e87d9d9 100644 --- a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp @@ -105,7 +105,10 @@ jvmtiIterationControl JNICALL ReferenceChainTracker::heapRootCallback( } if (ctx->tracker->_candidate_count > 0) { u32 klass_id = ctx->tracker->classTags()->resolve(class_tag); - ctx->tracker->recordDiscoveredInstance(klass_id, frontier_tag, true); + ctx->tracker->recordDiscoveredInstance( + klass_id, frontier_tag, true, + CacheInvalidationMode::DEFERRED, ctx->resolved_chain_tags, + ctx->resolved_chain_tag_count); } // Queue for expandFrontier() - the plain backlog lane, mirroring admitObject()'s // non-priority push tail. @@ -118,9 +121,14 @@ jvmtiIterationControl JNICALL ReferenceChainTracker::heapRootCallback( } break; } - // Prefer the more durable root when an object is rediscovered. - ctx->tracker->maybeUpgradeRootAttachedRootKind(ctx->frontier, *tag_ptr, - translated_root_kind); + // Prefer the more durable root when an object is rediscovered and invalidate a cached chain so + // its root attribution is rebuilt. + if (ctx->tracker->maybeUpgradeRootAttachedRootKind( + ctx->frontier, *tag_ptr, translated_root_kind)) { + ctx->tracker->deferResolvedChainInvalidation( + *tag_ptr, ctx->resolved_chain_tags, + ctx->resolved_chain_tag_count); + } break; default: break; @@ -187,8 +195,11 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, // comment) - so even when it runs this pass, the expandFrontier() call below is still needed to // make any further progress. if (run_root_enum) { + jlong resolved_chain_tags[MAX_RESOLVED_CHAINS]; ReferenceChainPassContext ctx; ctx.tracker = this; + ctx.resolved_chain_tags = resolved_chain_tags; + ctx.resolved_chain_tag_count = snapshotResolvedChainTags(resolved_chain_tags); ctx.frontier = _frontier; ctx.hop_cap = _hop_cap; ctx.budget = root_enum_budget; @@ -429,6 +440,9 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, // last one that ran. *truncated = *truncated || rotation_truncated; *frontier_cap_hit = *frontier_cap_hit || rotation_frontier_cap_hit; + + // Deferred chain invalidations are applied by the cache readers, + // pollWatchedTargets() and drainPendingChainEvents(). } // Incremental resumption across passes. @@ -455,6 +469,7 @@ void ReferenceChainTracker::expandFrontier(jvmtiEnv *jvmti, JNIEnv *jni, "GetObjectsWithTags/FollowReferences are JVMTI Heap-category calls " "and must not be made from GarbageCollectionStart/Finish"); + jlong resolved_chain_tags[MAX_RESOLVED_CHAINS]; ReferenceChainPassContext ctx; ctx.tracker = this; ctx.frontier = _frontier; @@ -463,6 +478,8 @@ void ReferenceChainTracker::expandFrontier(jvmtiEnv *jvmti, JNIEnv *jni, ctx.edges_admitted = 0; ctx.truncated = false; ctx.frontier_cap_hit = false; + ctx.resolved_chain_tags = resolved_chain_tags; + ctx.resolved_chain_tag_count = snapshotResolvedChainTags(resolved_chain_tags); // ARRAY-HOLDER BATCHING: expand a whole batch of boundary objects with ONE // FollowReferences(initial_object=holder_array) call per BFS level, instead of one @@ -855,6 +872,7 @@ void ReferenceChainTracker::admitStaticFieldRoots(jvmtiEnv *jvmti, JNIEnv *jni, return; } + jlong resolved_chain_tags[MAX_RESOLVED_CHAINS]; ReferenceChainPassContext ctx; ctx.tracker = this; ctx.frontier = _frontier; @@ -863,6 +881,8 @@ void ReferenceChainTracker::admitStaticFieldRoots(jvmtiEnv *jvmti, JNIEnv *jni, ctx.edges_admitted = 0; ctx.truncated = false; ctx.frontier_cap_hit = false; + ctx.resolved_chain_tags = resolved_chain_tags; + ctx.resolved_chain_tag_count = snapshotResolvedChainTags(resolved_chain_tags); // Empty (not null) batch_tags forces heapReferenceCallback() to stop at exactly one hop past each // class - see this method's own header comment for why a deeper descent here would reintroduce // the whole-graph FollowReferences cost the array-holder batching design otherwise avoids. diff --git a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp index a9bed3f3fd..320e9048b9 100644 --- a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp @@ -325,7 +325,10 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( // noise slots (see recordDiscoveredInstance). if (ctx->tracker->_candidate_count > 0) { u32 klass_id = ctx->tracker->classTags()->resolve(class_tag); - ctx->tracker->recordDiscoveredInstance(klass_id, frontier_tag, true); + ctx->tracker->recordDiscoveredInstance( + klass_id, frontier_tag, true, + CacheInvalidationMode::DEFERRED, ctx->resolved_chain_tags, + ctx->resolved_chain_tag_count); } if (ctx->batch_tags != nullptr) { // Same batching gate as the ordinary admission path below: under array-holder batching, @@ -411,8 +414,9 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( for (int s = 0; s < ctx->tracker->_candidate_count; s++) { if (ctx->tracker->_candidate_klass_ids[s] == klass_id) { matched = true; - ctx->tracker->recordDiscoveredInstance(klass_id, *tag_ptr, - false); + ctx->tracker->recordDiscoveredInstance( + klass_id, *tag_ptr, false, CacheInvalidationMode::DEFERRED, + ctx->resolved_chain_tags, ctx->resolved_chain_tag_count); break; } } @@ -452,7 +456,10 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( (u8)reference_kind)) { // Chain was improved — invalidate any cached chain for this tag so pollWatchedTargets // rebuilds it with the deeper path. - ctx->tracker->invalidateResolvedChain(*tag_ptr); + // Deferred: we're inside the FollowReferences pause (see deferResolvedChainInvalidation()). + ctx->tracker->deferResolvedChainInvalidation( + *tag_ptr, ctx->resolved_chain_tags, + ctx->resolved_chain_tag_count); if (was_root_attached_durable) { // Demotion push (B'): the replaced entry's static/JNI-global attribution was its only // anchor-tier eligibility, and it is gone now. @@ -465,7 +472,9 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( // Equal-depth re-parent from a transient root to a durable one (improveChain() cannot // express it - see its declaration) - same cache invalidation so the rebuilt chain uses the // durable root. - ctx->tracker->invalidateResolvedChain(*tag_ptr); + ctx->tracker->deferResolvedChainInvalidation( + *tag_ptr, ctx->resolved_chain_tags, + ctx->resolved_chain_tag_count); } } else { // Already-admitted entry reached via a NEW root-like edge (parent_tag == 0): the static-field @@ -475,7 +484,9 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( if (ctx->tracker->maybeUpgradeRootAttachedRootKind(ctx->frontier, *tag_ptr, (u8)reference_kind)) { - ctx->tracker->invalidateResolvedChain(*tag_ptr); + ctx->tracker->deferResolvedChainInvalidation( + *tag_ptr, ctx->resolved_chain_tags, + ctx->resolved_chain_tag_count); } else if (reference_kind == JVMTI_HEAP_REFERENCE_STATIC_FIELD) { // The upgrade refused (maybeUpgradeRootAttachedRootKind returns false for parent_tag != 0 // by design), so this STATIC_FIELD edge just proved an at-risk static attachment the anchor diff --git a/ddprof-lib/src/main/cpp/referenceChains.cpp b/ddprof-lib/src/main/cpp/referenceChains.cpp index a3c90849af..c755b119e1 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.cpp +++ b/ddprof-lib/src/main/cpp/referenceChains.cpp @@ -254,6 +254,7 @@ Error ReferenceChainTracker::start(Arguments &args) { ExclusiveLockGuard guard(&_resolved_chains_lock); _resolved_chains.clear(); } + clearPendingChainInvalidations(); { ExclusiveLockGuard guard(&_pending_abandoned_events_lock); _pending_abandoned_events.clear(); @@ -433,6 +434,7 @@ void ReferenceChainTracker::stop() { ExclusiveLockGuard guard(&_resolved_chains_lock); _resolved_chains.clear(); } + clearPendingChainInvalidations(); { ExclusiveLockGuard guard(&_pending_abandoned_events_lock); _pending_abandoned_events.clear(); @@ -579,7 +581,15 @@ void ReferenceChainTracker::threadLoop() { _oom_ramp_active = urgent; LivenessTracker::instance()->setUrgentTracking(urgent); - bool should_run = shouldRunPass(now_ns); + // shouldRunPass()'s restart branch (restartSearch()) clears _pending_expand/_priority_expand + // the same way a pass does - take _engine_lock around the call so that mutation is serialized + // with runPassSerialized()/finishLoopIterationSerialized() and with the pendingExpand*ForTest() + // read seams, instead of racing them. + bool should_run; + { + MutexLocker engine_guard(_engine_lock); + should_run = shouldRunPass(now_ns); + } // Only sleep when idle (no pass will run). When a canary search is active or a pass is about to // run, skip the sleep to run passes back-to-back. if (!should_run && cadence_ns > 0) { @@ -861,6 +871,7 @@ void ReferenceChainTracker::restartSearch() { if (_frontier != nullptr) { _frontier->resetForRestart(); } + clearPendingChainInvalidations(); _next_tag = 1; // Hop-edge label cache: keyed by raw class tags, which survive a restart (the shared class-tag // allocator is deliberately not reset - see this method's own declaration comment) - but the @@ -1014,6 +1025,7 @@ void ReferenceChainTracker::resetSearchStateForTest(jvmtiEnv *jvmti, ExclusiveLockGuard guard(&_resolved_chains_lock); _resolved_chains.clear(); } + clearPendingChainInvalidations(); { ExclusiveLockGuard guard(&_pending_abandoned_events_lock); _pending_abandoned_events.clear(); @@ -1029,6 +1041,10 @@ long ReferenceChainTracker::pendingExpandPositionForTest(jlong tag) const { if (tag == 0) { return -2; } + // _pending_expand/_priority_expand are otherwise only touched by the BFS thread under + // _engine_lock (runPass()/expandFrontier(), and shouldRunPass()'s restart branch) - take the + // same lock here so this read-only snapshot cannot observe either deque mid-mutation. + MutexLocker engine_guard(_engine_lock); // _priority_expand drains first (expandFrontier()'s own comment), so its entries are reported as // coming before _pending_expand's. long pos = 0; @@ -1048,6 +1064,7 @@ long ReferenceChainTracker::pendingExpandPositionForTest(jlong tag) const { } size_t ReferenceChainTracker::pendingExpandSizeForTest() const { + MutexLocker engine_guard(_engine_lock); return _pending_expand.size() + _priority_expand.size(); } @@ -1138,7 +1155,8 @@ jlong ReferenceChainTracker::tagAsRootForTest(jvmtiEnv *jvmti, JNIEnv *jni, } // Discovery recording for this seam's decoupling contract. if (_candidate_count > 0) { - recordDiscoveredInstance(klass_id, tag, false); + recordDiscoveredInstance(klass_id, tag, false, + CacheInvalidationMode::SYNCHRONOUS, nullptr, 0); } return tag; } diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index d360ea23f0..6e1da0dac8 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -389,24 +389,35 @@ class ReferenceChainTracker { public: bool contains(jlong tag) const { - u64 i = mix(tag) >> (64 - SLOT_SHIFT); + const u64 start = mix(tag) >> (64 - SLOT_SHIFT); + u64 i = start; while (_used[i]) { if (_keys[i] == tag) { return true; } i = (i + 1) & SLOT_MASK; + if (i == start) { + return false; + } } return false; } // Idempotent: returns false if `tag` is already indexed. + // Returns false on a full table instead of probing forever; callers + // already cap the deque at PRIORITY_EXPAND_CAP. bool insert(jlong tag) { - u64 i = mix(tag) >> (64 - SLOT_SHIFT); + const u64 start = mix(tag) >> (64 - SLOT_SHIFT); + u64 i = start; while (_used[i]) { if (_keys[i] == tag) { return false; } i = (i + 1) & SLOT_MASK; + // Wrapped around: table full. + if (i == start) { + return false; + } } _used[i] = 1; _keys[i] = tag; @@ -584,6 +595,15 @@ class ReferenceChainTracker { std::unordered_map _resolved_chains; SpinLock _resolved_chains_lock; + // Chain evictions deferred by heap callbacks. Only tags present in the immutable pre-traversal + // resolved-cache snapshot are eligible, so distinct pending tags are bounded exactly by the + // cache cap. Both buffers are reserved up front: drains swap the active producer buffer into the + // scratch buffer under the pending lock, then erase cache entries after releasing that lock. + static constexpr int MAX_PENDING_CHAIN_INVALIDATIONS = MAX_RESOLVED_CHAINS; + std::vector _pending_chain_invalidations; + std::vector _draining_chain_invalidations; + SpinLock _pending_chain_invalidations_lock; + // Abandoned-search events awaiting Profiler::dump() (profiler.cpp). static constexpr int MAX_PENDING_ABANDONED_EVENTS = 16; std::vector _pending_abandoned_events; @@ -604,9 +624,10 @@ class ReferenceChainTracker { // of each pass - see runPass()'s own comment). PainBudget _cpu_pain_budget; - // The cache above is mutated on this tracker's own BFS scheduling thread (pollWatchedTargets()) - // and read on whatever thread calls Profiler::dump() (drainPendingChainEvents()); - // _resolved_chains_lock (declared with the cache) is the only synchronization between them. + // The engine/BFS thread inserts and refreshes the cache. A dump thread may snapshot it and drain + // exact deferred entries while a traversal is active. _resolved_chains_lock protects every cache + // access and serializes use of _draining_chain_invalidations; callbacks take only the pending + // lock. Drains acquire the resolved-cache lock before the pending lock. // Fallback cadence between passes. static constexpr u64 PASS_CADENCE_NS = 1000000000ULL; // 1s @@ -741,7 +762,10 @@ class ReferenceChainTracker { _stale_expanded_rotation_cursor(1), _thread_walk_anchor_cursor(0), _safepoint_pain_budget(0.0), _search_pain_ms(0), _cpu_pain_budget(0.0), - _thread(), _running(false), _abort_pass_requested(false) {} + _thread(), _running(false), _abort_pass_requested(false) { + _pending_chain_invalidations.reserve(MAX_PENDING_CHAIN_INVALIDATIONS); + _draining_chain_invalidations.reserve(MAX_PENDING_CHAIN_INVALIDATIONS); + } void onGCStart(); void onGCFinish(); @@ -861,8 +885,16 @@ class ReferenceChainTracker { void buildDiscoveredInstanceChains(jvmtiEnv *jvmti, JNIEnv *jni, u32 klass_id, u64 current_search_ns); + enum class CacheInvalidationMode : u8 { + SYNCHRONOUS, + DEFERRED + }; + void recordDiscoveredInstance(u32 klass_id, jlong frontier_tag, - bool leak_correlated); + bool leak_correlated, + CacheInvalidationMode invalidation_mode, + const jlong *resolved_chain_tags, + int resolved_chain_tag_count); // Correlate a leak tag with an instance the BFS admitted BEFORE tagLeakInstances() tagged it (its // JVMTI tag is a frontier tag, its frontier entry has leak_tag == 0). @@ -990,6 +1022,22 @@ class ReferenceChainTracker { // Remove a cached chain so pollWatchedTargets rebuilds it on the next poll. void invalidateResolvedChain(jlong source_tag); + // Copies the current cache keys into fixed caller-owned storage before a synchronous traversal. + int snapshotResolvedChainTags(jlong *out); + + // Queues an eviction from a heap callback only when the tag was cached in the immutable + // pre-traversal snapshot. This never takes _resolved_chains_lock or allocates. + void deferResolvedChainInvalidation(jlong source_tag, + const jlong *resolved_chain_tags, + int resolved_chain_tag_count); + + // Applies queued exact-tag evictions. Called by the engine thread and by dump threads, which may + // overlap a traversal; the resolved-cache/pending locks provide synchronization. + void drainPendingChainInvalidations(); + + // Clears both pending buffers at cache/tag-generation boundaries without releasing capacity. + void clearPendingChainInvalidations(); + // Snapshots the just-abandoned search into _pending_abandoned_events - called from runPass() // (referenceChains.cpp) immediately after it writes SearchState::ABANDONED, while // buildAbandonedEvent()'s source fields are still valid (see _pending_abandoned_events' own @@ -1154,8 +1202,12 @@ class ReferenceChainTracker { // JVMTI tag helpers used by the heap-walk callbacks. jlong nextTag() { return atomicIncRelaxed(_next_tag, (jlong)1); } - // Held by runPassSerialized() and finishLoopIterationSerialized() around the engine work. - Mutex _engine_lock; + // Held by runPassSerialized() and finishLoopIterationSerialized() around the engine work, by + // threadLoop() around shouldRunPass() (whose restart branch mutates _pending_expand/ + // _priority_expand the same way a pass does), and by the pendingExpand*ForTest() seams below so + // they never observe those deques mid-mutation from the BFS thread. Mutable because the test + // seams are const (read-only from the caller's perspective) but still need to take the lock. + mutable Mutex _engine_lock; jlong tagObject(jvmtiEnv *jvmti, jobject obj); jlong getTag(jvmtiEnv *jvmti, jobject obj); void clearTag(jvmtiEnv *jvmti, jobject obj); @@ -1316,15 +1368,16 @@ class ReferenceChainTracker { // for computing a position's fraction of the current backlog. size_t pendingExpandSizeForTest() const; - // Test seam - not part of the production API. Exposes the private shouldRunPass() gate directly, - // so a test can assert whether a fresh/terminal search would be allowed to start right now - in - // particular, whether LivenessTracker::secondsToOOM()'s urgent-OOM bypass (hasLeakSignal(), see - // OOM_URGENT_THRESHOLD_S's own comment above) opens this gate even with zero per-klass leak - // candidate (confirmable in the same test via - // LivenessTracker::selectLeakCandidates()/JavaProfiler's selectLeakCandidateKlassIds0() seam) - - // something runReferenceChainPass0() (javaApi.cpp) cannot show, since it calls runPass() directly - // and never consults this gate at all. - bool shouldRunPassForTest(u64 now_ns) { return shouldRunPass(now_ns); } + // Test seam - not part of the production API. Evaluates the real private scheduling gate without + // executing a BFS pass. This is not read-only: while holding _engine_lock like threadLoop(), the + // gate may account a completed search's pain and restart/reset terminal search state. It also + // exposes whether LivenessTracker::secondsToOOM()'s urgent-OOM bypass opens the gate even with no + // per-klass candidate; runReferenceChainPass0() cannot show that because it calls runPass() + // directly and never consults the gate. + bool shouldRunPassForTest(u64 now_ns) { + MutexLocker engine_guard(_engine_lock); + return shouldRunPass(now_ns); + } }; #endif // _REFERENCECHAINS_H diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index c65d80b00d..15d7601f9f 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -462,6 +462,14 @@ public Map getDebugCounters() { private static native int getTid0(); + /** + * Returns the calling thread's profiler tid. Tests seeding per-tid trends + * via {@link #seedTidTrendSample0} must call this on the leaking thread. + */ + static int getTid() { + return getTid0(); + } + private static native boolean recordTrace0(long rootSpanId, String endpoint, String operation, int sizeLimit); private static native void dump0(String recordingFilePath); @@ -530,6 +538,111 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l */ public static native boolean testTlsPrimingAvailable(); + /** + * Test seam. The test seams below are implemented in debug native builds only; + * in release builds they are no-ops. + *

+ * Enables/disables per-klass population tracking without a live-JVM + * {@code initialize()}. Returns {@code true} on debug builds. + */ + static native boolean setGcGenerationsEnabled0(boolean enabled); + + /** + * Test seam: seeds one epoch of population history for {@code klassId}. + */ + static native void seedKlassPopulationSample0(int klassId, int count, long epoch); + + /** + * Test seam: seeds one epoch of per-tid trend history for {@code klassId}. + * {@code tid} should come from {@link #getTid()}. + */ + static native void seedTidTrendSample0(int klassId, int tid, int count, long epoch); + + /** + * Test seam: makes {@code representative} the leak-candidate representative + * of {@code klassId}. + */ + static native void setKlassPopulationRepresentativeForTest0(int klassId, Object representative); + + /** + * Test seam: clears the per-klass population table. + */ + static native void resetKlassPopulationForTest0(); + + /** + * Test seam: the klass ids currently selected as leak candidates (top 5). + */ + static native int[] selectLeakCandidateKlassIds0(); + + /** + * Test seam: tags {@code target} as a reference-chain frontier root. + * Returns the frontier tag, or {@code 0} on failure. + */ + static native long tagAsReferenceChainRoot0(Object target); + + /** + * Test seam: runs one reference-chain search pass synchronously. Returns + * {@code false} if reference chains are disabled or not started. + */ + static native boolean runReferenceChainPass0(); + + /** + * Test seam: runs one poll of the leak-candidate to chain bridge synchronously. + */ + static native void pollReferenceChainTargets0(); + + /** + * Test seam: drains queued reference-chain events and returns their count. + */ + static native int drainReferenceChainEventCount0(); + + /** + * Test seam: resets the search state and releases held tags, so the next + * pass walks from the roots again. + */ + static native void resetReferenceChainSearchForTest0(); + + /** + * Test seam: position of {@code target} in the pending-expansion queue + * ({@code 0} = next), {@code -1} if already expanded, {@code -2} if never + * admitted. Does not tag {@code target}. + */ + static native long getReferenceChainPendingPositionForTest0(Object target); + + /** + * Test seam: size of the pending-expansion queue. + */ + static native long getReferenceChainPendingSizeForTest0(); + + /** + * Test seam: seeds one heap-floor sample for the time-to-OOM projection. + * Timestamps only need to be increasing. + */ + static native void heapFloorRecordForTest0(long usedBytes, long timestampNs); + + /** + * Test seam: overrides the max heap size used by the time-to-OOM projection. + */ + static native void setMaxHeapBytesForTest0(long maxHeapBytes); + + /** + * Test seam: {@code false} stops GCs from recording heap-floor samples, + * {@code true} restores it. + */ + static native void setHeapFloorRecordingForTest0(boolean enabled); + + /** + * Test seam: evaluates the real search scheduling gate without executing a BFS pass. + * This is not a read-only query: under the native engine lock it may account a completed + * search's pain and restart/reset terminal search state. + */ + static native boolean shouldRunPassForTest0(); + + /** + * Test seam: number of passes run for the current search. + */ + static native int referenceChainPassesRunForTest0(); + // ---- Test-only reads of the current thread's OTEP record ---------------------------------- // Each resolves the current carrier's record directly (like the write primitives above) with // no cached buffer and no per-thread Java object; introspection/test use only. diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp index 9c0d76d2e3..cb59c87ca5 100644 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp @@ -17,9 +17,11 @@ #include #include "livenessTracker.h" #include "arguments.h" +#include "counters.h" #include "objectSampler.h" #include "profiler.h" #include "referenceChainsTestAccessors.h" +#include "vmEntry.h" #include "../../main/cpp/gtest_crash_handler.h" #include #include @@ -514,6 +516,34 @@ TEST_F(KlassPopulationTest, EvictsLeastRecentlyUpdatedEntryWhenFull) { EXPECT_EQ(new_entry.ring_fill, 1); } +TEST_F(KlassPopulationTest, TidTrendCreationReturnsEvictedRepresentatives) { + LivenessTracker *tracker = LivenessTracker::instance(); + const int cap = 256; + for (u32 klass_id = 1; klass_id <= (u32)cap; klass_id++) { + int slot; + bool created; + tracker->klassPopulationRecordForTest(klass_id, 1, klass_id, &slot, + &created); + ASSERT_TRUE(created); + } + jweak victim_ref = fakeRef(0xbeef); + tracker->klassPopulationSetRepresentativeForTest(nullptr, 1, victim_ref); + + jweak evicted[KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS]; + int evicted_count = 0; + tracker->tidTrendRecordForTest( + /*klass_id=*/cap + 1, /*tid=*/7, /*count=*/12, /*epoch=*/cap + 1, + evicted, &evicted_count, + KlassPopulationEntry::MAX_REPRESENTATIVES_PER_KLASS); + + ASSERT_EQ(1, evicted_count); + EXPECT_EQ(victim_ref, evicted[0]); + EXPECT_EQ(cap, tracker->klassPopulationSizeForTest()); + KlassPopulationEntry replacement{}; + EXPECT_TRUE(tracker->klassPopulationLookupForTest(cap + 1, + &replacement)); +} + // --------------------------------------------------------------------------- // Slope computation and candidate ranking. Same rationale as KlassPopulationTest above // for exercising LivenessTracker::instance() directly: selectLeakCandidates() @@ -1092,8 +1122,181 @@ class LivenessTrackerTestAccessor { #endif return recorded; } + + static void setTableCapsForTest(LivenessTracker *t, int table_size, + int table_max_cap) { + t->_table_size = table_size; + t->_table_max_cap = table_max_cap; + } + + static int tableSizeFieldForTest(LivenessTracker *t) { + return t->_table_size; + } + + static int tableMaxCapForTest(LivenessTracker *t) { + return t->_table_max_cap; + } + + static void callCleanupTableForTest(LivenessTracker *t, bool forced, + bool allow_resolve, bool account_epoch) { + t->cleanup_table(forced, allow_resolve, account_epoch); + } + + static void setGcEpochForTest(LivenessTracker *t, u64 epoch) { + t->_gc_epoch = epoch; + } + + static void setLastGcEpochForTest(LivenessTracker *t, u64 epoch) { + t->_last_gc_epoch = epoch; + } + + static u64 lastGcEpochForTest(LivenessTracker *t) { + return t->_last_gc_epoch; + } + + static void setSessionGatesForTest(LivenessTracker *t, bool enabled, + bool generations) { + t->_enabled = enabled; + t->_gc_generations.store(generations, std::memory_order_relaxed); + } + + static bool enabledForTest(LivenessTracker *t) { + return t->_enabled; + } + + static bool entryAgeAndAdmissionForTest(LivenessTracker *t, u32 idx, + jlong *age, + u64 *admission_epoch) { + if (t->_table == nullptr || idx >= t->_table_size || + __atomic_load_n(&t->_table[idx].ready, __ATOMIC_ACQUIRE) != 1) { + return false; + } + *age = t->_table[idx].age; + *admission_epoch = t->_table[idx].admission_epoch; + return true; + } }; +TEST(LivenessTrackerSessionTest, ResetSessionClearsOnlyRecordingGates) { + LivenessTracker *tracker = LivenessTracker::instance(); + LivenessTrackerTestAccessor::setSessionGatesForTest( + tracker, /*enabled=*/true, /*generations=*/true); + tracker->resetSession(); + EXPECT_FALSE(LivenessTrackerTestAccessor::enabledForTest(tracker)); + EXPECT_FALSE(tracker->gcGenerationsEnabled()); +} + +// cleanup_table() calls VM::jni(), so install a fake JavaVM. +static JNIEnv_ g_cleanup_table_mock_jni_env{}; + +static jint JNICALL cleanupTableMockGetEnv(JavaVM *, void **penv, jint) { + *penv = &g_cleanup_table_mock_jni_env; + return 0; // JNI_OK +} + +// Empty table, so cleanup_table() makes no JNI calls beyond VM::jni(). +class CleanupTableAccountEpochTest : public ::testing::Test { +protected: + JNIInvokeInterface_ vm_tbl{}; + JavaVM_ mock_vm{}; + JavaVM *orig_vm = nullptr; + int saved_table_size = 0; + int saved_table_max_cap = 0; + + void SetUp() override { + installGtestCrashHandler(); + orig_vm = VMTestAccessor::getVm(); + vm_tbl = JNIInvokeInterface_{}; + vm_tbl.GetEnv = &cleanupTableMockGetEnv; + mock_vm.functions = &vm_tbl; + VMTestAccessor::setVm(&mock_vm); + + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(true); + LivenessTracker::instance()->classMapGenerationSetForTest( + Profiler::instance()->classMap()->generation()); + saved_table_size = LivenessTrackerTestAccessor::tableSizeFieldForTest( + LivenessTracker::instance()); + saved_table_max_cap = LivenessTrackerTestAccessor::tableMaxCapForTest( + LivenessTracker::instance()); + LivenessTrackerTestAccessor::setTableCapsForTest( + LivenessTracker::instance(), /*table_size=*/0, /*table_max_cap=*/0); + } + + void TearDown() override { + VMTestAccessor::setVm(orig_vm); + LivenessTrackerTestAccessor::setGcEpochForTest(LivenessTracker::instance(), 0); + LivenessTrackerTestAccessor::setLastGcEpochForTest(LivenessTracker::instance(), 0); + LivenessTracker::instance()->klassPopulationResetForTest(); + LivenessTracker::instance()->setGcGenerationsForTest(false); + LivenessTrackerTestAccessor::setTableCapsForTest( + LivenessTracker::instance(), saved_table_size, saved_table_max_cap); + restoreDefaultSignalHandlers(); + } +}; + +TEST_F(CleanupTableAccountEpochTest, AccountEpochFalseSkipsEpochClaim) { + LivenessTracker *tracker = LivenessTracker::instance(); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 5); + ASSERT_EQ(0u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); + + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, /*account_epoch=*/false); + EXPECT_EQ(0u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)) + << "account_epoch=false must not claim the GC epoch"; + + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, /*account_epoch=*/false); + EXPECT_EQ(0u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); +} + +TEST_F(CleanupTableAccountEpochTest, AccountEpochTrueClaimsEpoch) { + LivenessTracker *tracker = LivenessTracker::instance(); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 5); + ASSERT_EQ(0u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); + + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, /*account_epoch=*/true); + EXPECT_EQ(5u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)) + << "account_epoch=true must claim the newer GC epoch"; +} + +TEST_F(CleanupTableAccountEpochTest, EpochOwnerStillClaimsAfterReaperSweep) { + LivenessTracker *tracker = LivenessTracker::instance(); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 7); + + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, /*account_epoch=*/false); + ASSERT_EQ(0u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); + + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, /*account_epoch=*/true); + EXPECT_EQ(7u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); +} + +TEST_F(CleanupTableAccountEpochTest, EpochOwnerFoldsKnownKlassOnEmptyTable) { + LivenessTracker *tracker = LivenessTracker::instance(); + int slot; + bool created; + tracker->klassPopulationRecordForTest(42, /*count=*/9, /*epoch=*/4, + &slot, &created); + ASSERT_TRUE(created); + LivenessTrackerTestAccessor::setLastGcEpochForTest(tracker, 4); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 5); + + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, + /*account_epoch=*/true); + + EXPECT_EQ(5u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); + KlassPopulationEntry entry{}; + ASSERT_TRUE(tracker->klassPopulationLookupForTest(42, &entry)); + EXPECT_EQ(5u, entry.last_updated_epoch); + ASSERT_EQ(2, entry.ring_fill); + u8 newest = (u8)((entry.ring_head + 30 - 1) % 30); + EXPECT_EQ(0u, entry.count_ring[newest]); +} + class SecondsToOOMTest : public ::testing::Test { protected: static constexpr u64 SEC_NS = 1000000000ULL; @@ -1126,7 +1329,7 @@ class SecondsToOOMTest : public ::testing::Test { }; // Fewer than KLASS_POPULATION_MIN_FILL_FOR_TREND (10) heap-floor samples - -// same "not enough history yet" gate ringThirdsStats() applies to every +// same "not enough history yet" gate ringWindowStats() applies to every // other trend check in this class. TEST_F(SecondsToOOMTest, NotEnoughSamplesReturnsNegative) { LivenessTracker *tracker = LivenessTracker::instance(); @@ -1207,6 +1410,39 @@ TEST_F(SecondsToOOMTest, RisingFloorProjectsExpectedSeconds) { EXPECT_NEAR(tracker->secondsToOOM(), 9.0, 1e-6); } +// Dip-then-recover: the recent half rises but the full-window delta is 0, so +// there must be no projection (and no +inf). No container limit is set. +TEST_F(SecondsToOOMTest, DipThenRecoverFloorReturnsNegativeNotInf) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(2800 * MiB)); + // Symmetric V: the fitted slope is exactly 0. + static const long shape[] = {1900, 1800, 1700, 1600, 1500, + 1500, 1600, 1700, 1800, 1900}; + for (int i = 0; i < 10; i++) { + tracker->heapFloorRecordForTest((u64)shape[i] * MiB, (u64)i * SEC_NS); + } + double secs = tracker->secondsToOOM(); + EXPECT_LT(secs, 0.0) + << "a dip-then-recover window must not offer a projection"; + EXPECT_TRUE(secs > -1e18 && secs < 1e18) + << "the projection must be finite, never +inf"; +} + +// Odd-length window: fill/2 rounds down, so the recent half excludes the +// median sample; a steady rise must still project. +TEST_F(SecondsToOOMTest, OddLengthRisingWindowStillProjects) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setMaxHeapBytesForTest((jlong)(3000 * MiB)); + // 1000..2000MiB, 100MiB/s. + for (int i = 0; i < 11; i++) { + tracker->heapFloorRecordForTest(1000 * MiB + (u64)i * 100 * MiB, + (u64)i * SEC_NS); + } + double secs = tracker->secondsToOOM(); + EXPECT_GT(secs, 0.0); + EXPECT_NEAR(secs, 9.0, 1.5); +} + // The floor's own recent-third mean has already reached the max heap size - // exhaustion is "now", not some positive number of seconds out. TEST_F(SecondsToOOMTest, FloorAtMaxHeapReturnsZero) { @@ -1664,6 +1900,55 @@ TEST_F(AdmissionBoostTest, ZeroCandidatePollClearsWatchedSet) { EXPECT_FALSE(tracker->admitForTrackingForTest(77)); } +// At the table cap the urgency boost stops admitting everything; watched tids +// are still admitted. +TEST_F(AdmissionBoostTest, VolumeBackstopCapsUrgencyAdmission) { + LivenessTracker *tracker = LivenessTracker::instance(); + tracker->setUrgentTracking(true); + int saved_table_size = LivenessTrackerTestAccessor::tableSizeFieldForTest(tracker); + int saved_table_max_cap = LivenessTrackerTestAccessor::tableMaxCapForTest(tracker); + + // max_cap == 0 disables the backstop. + LivenessTrackerTestAccessor::setTableCapsForTest(tracker, /*table_size=*/999, + /*table_max_cap=*/0); + long long admits = Counters::getCounter(LIVENESS_URGENT_BOOST_ADMITS); + long long backed_off = Counters::getCounter(LIVENESS_URGENT_BOOST_BACKED_OFF); + EXPECT_TRUE(tracker->admitForTrackingForTest(1)); + EXPECT_EQ(admits + 1, Counters::getCounter(LIVENESS_URGENT_BOOST_ADMITS)); + EXPECT_EQ(backed_off, Counters::getCounter(LIVENESS_URGENT_BOOST_BACKED_OFF)); + + // Below the cap. + LivenessTrackerTestAccessor::setTableCapsForTest(tracker, /*table_size=*/5, + /*table_max_cap=*/10); + admits = Counters::getCounter(LIVENESS_URGENT_BOOST_ADMITS); + EXPECT_TRUE(tracker->admitForTrackingForTest(2)); + EXPECT_EQ(admits + 1, Counters::getCounter(LIVENESS_URGENT_BOOST_ADMITS)); + EXPECT_EQ(backed_off, Counters::getCounter(LIVENESS_URGENT_BOOST_BACKED_OFF)); + admits = Counters::getCounter(LIVENESS_URGENT_BOOST_ADMITS); + + // At the cap: falls back to the ratio gate, which is 0 here. + LivenessTrackerTestAccessor::setTableCapsForTest(tracker, /*table_size=*/10, + /*table_max_cap=*/10); + EXPECT_FALSE(tracker->admitForTrackingForTest(3)); + EXPECT_EQ(admits, Counters::getCounter(LIVENESS_URGENT_BOOST_ADMITS)); + EXPECT_EQ(backed_off + 1, Counters::getCounter(LIVENESS_URGENT_BOOST_BACKED_OFF)); + + // Above the cap. + LivenessTrackerTestAccessor::setTableCapsForTest(tracker, /*table_size=*/11, + /*table_max_cap=*/10); + EXPECT_FALSE(tracker->admitForTrackingForTest(4)); + EXPECT_EQ(backed_off + 2, Counters::getCounter(LIVENESS_URGENT_BOOST_BACKED_OFF)); + + // A watched tid is still admitted. + KlassCandidate kc; + fillCandidate(&kc, 5); + tracker->noteSelectedCandidates(&kc, 1); + EXPECT_TRUE(tracker->admitForTrackingForTest(5)); + + LivenessTrackerTestAccessor::setTableCapsForTest(tracker, saved_table_size, + saved_table_max_cap); +} + // --------------------------------------------------------------------------- // C2 chaos: hammer admitForTracking() from many threads while a publisher // thread flips the boost state (noteSelectedCandidates()/setUrgentTracking()) @@ -2183,6 +2468,44 @@ class LivenessTrackerMockJvmTest : public ::testing::Test { u64 LivenessTrackerMockJvmTest::fake_now_ns = 0; +TEST_F(LivenessTrackerMockJvmTest, SurvivorAgingStartsAtAdmissionEpoch) { + LivenessTracker *tracker = LivenessTracker::instance(); + ASSERT_EQ(0u, tracker->tableSizeForTest()); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 0); + LivenessTrackerTestAccessor::setLastGcEpochForTest(tracker, 0); + + trackNew(&jvm.retained_class, /*tid=*/1); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 1); + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, + /*account_epoch=*/true); + + jlong old_age = -1; + u64 old_admission = UINT64_MAX; + ASSERT_TRUE(LivenessTrackerTestAccessor::entryAgeAndAdmissionForTest( + tracker, 0, &old_age, &old_admission)); + EXPECT_EQ(0u, old_admission); + EXPECT_EQ(1, old_age); + + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 3); + trackNew(&jvm.retained_class, /*tid=*/1); + LivenessTrackerTestAccessor::setGcEpochForTest(tracker, 5); + LivenessTrackerTestAccessor::callCleanupTableForTest( + tracker, /*forced=*/true, /*allow_resolve=*/false, + /*account_epoch=*/true); + + jlong new_age = -1; + u64 new_admission = UINT64_MAX; + ASSERT_TRUE(LivenessTrackerTestAccessor::entryAgeAndAdmissionForTest( + tracker, 0, &old_age, &old_admission)); + ASSERT_TRUE(LivenessTrackerTestAccessor::entryAgeAndAdmissionForTest( + tracker, 1, &new_age, &new_admission)); + EXPECT_EQ(5, old_age); + EXPECT_EQ(3u, new_admission); + EXPECT_EQ(2, new_age) << "entry must not age for epochs before admission"; + EXPECT_EQ(5u, LivenessTrackerTestAccessor::lastGcEpochForTest(tracker)); +} + // Each sweep resolves the class of at most RESOLVE_BUDGET_PER_SWEEP (256, // livenessTracker.cpp) survivors. Survivors keep their table order across // compaction, so the budget must go to entries that have no cached class id diff --git a/ddprof-lib/src/test/cpp/objectSampler_ut.cpp b/ddprof-lib/src/test/cpp/objectSampler_ut.cpp index 0def85da5c..50cda8838b 100644 --- a/ddprof-lib/src/test/cpp/objectSampler_ut.cpp +++ b/ddprof-lib/src/test/cpp/objectSampler_ut.cpp @@ -33,8 +33,27 @@ class ObjectSamplerTestAccessor { jclass klass, jlong size) { s->recordAllocation(jvmti, jni, thread, event_type, object, klass, size); } + + static void setSessionFlags(ObjectSampler *s, bool allocations, + bool liveness, bool generations) { + s->_record_allocations = allocations; + s->_record_liveness = liveness; + s->_gc_generations = generations; + } + + static bool anySessionFlag(const ObjectSampler *s) { + return s->_record_allocations || s->_record_liveness || + s->_gc_generations; + } }; +TEST(ObjectSamplerTest, ResetSessionClearsRecordingMetadataFlags) { + ObjectSampler *sampler = ObjectSampler::instance(); + ObjectSamplerTestAccessor::setSessionFlags(sampler, true, true, true); + sampler->resetSession(); + EXPECT_FALSE(ObjectSamplerTestAccessor::anySessionFlag(sampler)); +} + // --------------------------------------------------------------------------- // Mock-JVMTI infrastructure for Deallocate regression tests // --------------------------------------------------------------------------- diff --git a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc index 54e9288318..2559036d1c 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc @@ -333,10 +333,21 @@ TEST_F(ReferenceChainsBfsTest, RecordDiscoveredInstanceEvictsNoiseSlots) { EXPECT_EQ((jlong)100, ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); - // Leak-correlated discovery evicts the first noise slot (tag 100 has no frontier entry -> - // treated as uncorrelated). - ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest( - kKlass, 200, true); + // Callback-mode leak correlation replaces the noise slot immediately but defers cache erasure + // through the immutable pre-traversal snapshot. + ReferenceChainEvent cached; + cached._target_tag = 100; + cached._depth = 0; + ReferenceChainsTestAccessor::cacheChain(100, cached, 100, 0); + std::vector snapshot = + ReferenceChainsTestAccessor::snapshotResolvedChainTags(); + ReferenceChainsTestAccessor::recordDiscoveredInstanceDeferredForTest( + kKlass, 200, true, snapshot); + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(100)); + EXPECT_EQ(1u, + ReferenceChainsTestAccessor::pendingChainInvalidationCount()); + ReferenceChainsTestAccessor::drainPendingChainInvalidations(); + EXPECT_FALSE(ReferenceChainsTestAccessor::hasResolvedChainForTag(100)); EXPECT_EQ(cap, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); EXPECT_EQ((jlong)200, ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); diff --git a/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc b/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc index fddf2f5fb0..e3e58058c9 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsLifecycleTests.inc @@ -223,6 +223,18 @@ TEST_F(ReferenceChainsLifecycleTest, ChurnDuringConcurrentPassesStaysConsistent) EXPECT_GE(tracker->gcFinishEpoch(), gc_finish_before + 1); EXPECT_GE(tracker->gcStartEpoch(), tracker->gcFinishEpoch()); + // Under the non-TSan configs at least one search must reach a terminal + // outcome - otherwise the churn exercised nothing. Under TSan the + // instrumented passes plus the search's initial 1s cadence can burn the + // whole kMaxPasses budget inside the first search (observed: 1 failure in + // 50 shuffled repeats, no TSan report, no crash), so a budget exhaustion + // there is a documented skip, not a product failure. +#if TSAN_ENABLED + if (total_passes.load(std::memory_order_relaxed) >= kMaxPasses && + restarts.load(std::memory_order_relaxed) == 0) { + GTEST_SKIP() << "TSan slowdown exhausted the pass budget before a search terminated"; + } +#endif EXPECT_GE(restarts.load(), 1) << "no search ever reached a terminal outcome"; // I2: the recording boundary clears the previous recording's resolved diff --git a/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc b/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc index e8be62ca72..c21cbf9936 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsOomTests.inc @@ -259,12 +259,12 @@ TEST_F(ReferenceChainsCanaryRefillTest, BackoffGateHoldsUnlessOomRampActive) { ReferenceChainsTestAccessor::setSearchStartedForTest(true); // Open chase with the backoff engaged: mult=4 x ema=100ms = 400ms - // spacing, and only 50ms since the last pass. + // spacing, and only 50ms since the last pass. The EMA is in ns. ReferenceChainsTestAccessor::setCandidateCountForTest(2); ReferenceChainsTestAccessor::setCandidateFoundBitsForTest(0x1); u64 now = 6000000000ULL; // 6s into the test's synthetic clock ReferenceChainsTestAccessor::setCanaryBackoffForTest( - 4, 100, now - 50 * 1000000ULL); + 4, 100ULL * 1000000ULL, now - 50 * 1000000ULL); ReferenceChainsTestAccessor::setOomRampActive(false); EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(now)); diff --git a/ddprof-lib/src/test/cpp/referenceChainsPesTests.inc b/ddprof-lib/src/test/cpp/referenceChainsPesTests.inc new file mode 100644 index 0000000000..57e850ec47 --- /dev/null +++ b/ddprof-lib/src/test/cpp/referenceChainsPesTests.inc @@ -0,0 +1,70 @@ +// PriorityExpandSet and ClassTagAllocator tests. + +TEST(PriorityExpandSetTest, InsertContainsClearRoundTrip) { + ReferenceChainsTestAccessor::pesClear(); + + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(42)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesInsert(42)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(42)); + // Already present. + EXPECT_FALSE(ReferenceChainsTestAccessor::pesInsert(42)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesInsert(43)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(43)); + + ReferenceChainsTestAccessor::pesClear(); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(42)) + << "clear must empty the set"; + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(43)); + + EXPECT_TRUE(ReferenceChainsTestAccessor::pesInsert(42)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(42)); + ReferenceChainsTestAccessor::pesClear(); +} + +TEST(PriorityExpandSetTest, RebuildFromMatchesQueueContents) { + ReferenceChainsTestAccessor::pesClear(); + + std::deque queue = {10, 20, 30}; + ReferenceChainsTestAccessor::pesRebuildFrom(queue); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(10)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(20)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(30)); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(40)); + + queue.pop_front(); + ReferenceChainsTestAccessor::pesRebuildFrom(queue); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(10)) + << "stale member after rebuildFrom"; + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(20)); + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(30)); + ReferenceChainsTestAccessor::pesClear(); +} + +// insert() on a full table must return false instead of probing forever. +TEST(PriorityExpandSetTest, InsertOnFullTableTerminatesWithoutLivelock) { + ReferenceChainsTestAccessor::pesClear(); + + const int capacity = 1 << 11; // 2048 slots (SLOT_SHIFT == 11) + for (int i = 0; i < capacity; i++) { + ASSERT_TRUE(ReferenceChainsTestAccessor::pesInsert(1000 + (jlong)i * 7919)) + << "insert must succeed until the table is full (i=" << i << ")"; + } + EXPECT_TRUE(ReferenceChainsTestAccessor::pesContains(1000)); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(987654321)); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesInsert(987654321)); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesInsert(1000)); + ReferenceChainsTestAccessor::pesClear(); + EXPECT_FALSE(ReferenceChainsTestAccessor::pesContains(1000)); +} + +TEST(ClassTagAllocatorTest, MintsStrictlyNegativeStrictlyDecreasingTags) { + ClassTagAllocator::resetForTest(); + jlong first = ClassTagAllocator::next(); + jlong second = ClassTagAllocator::next(); + EXPECT_LT(first, 0) << "class tags must be negative (heapReferenceCallback" + "() dispatches on the sign)"; + EXPECT_LT(second, first) << "tags must decrease monotonically (no reuse)"; + ClassTagAllocator::resetForTest(); + EXPECT_EQ(first, ClassTagAllocator::next()) + << "resetForTest must restart the sequence at the same value"; +} diff --git a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h index 0e248f765e..ef2a0d6817 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h +++ b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h @@ -14,6 +14,7 @@ #ifndef REFERENCE_CHAINS_TEST_ACCESSORS_H #define REFERENCE_CHAINS_TEST_ACCESSORS_H +#include #include #include @@ -93,6 +94,7 @@ class ReferenceChainsTestAccessor { t->_passes_since_last_candidate_progress = 0; t->_last_candidate_progress_mark = 0; t->_canary_stuck_restart_count = 0; + t->clearPendingChainInvalidations(); t->_resolved_chains.clear(); t->_safepoint_pain_budget = PainBudget(); t->_cpu_pain_budget = PainBudget(); @@ -240,6 +242,43 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::MAX_RESOLVED_CHAINS; } + static std::vector snapshotResolvedChainTags() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + jlong tags[ReferenceChainTracker::MAX_RESOLVED_CHAINS]; + int count = t->snapshotResolvedChainTags(tags); + return std::vector(tags, tags + count); + } + + static void deferResolvedChainInvalidation( + jlong tag, const std::vector &snapshot) { + ReferenceChainTracker::instance()->deferResolvedChainInvalidation( + tag, snapshot.data(), (int)snapshot.size()); + } + + static size_t pendingChainInvalidationCount() { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + t->_pending_chain_invalidations_lock.lock(); + size_t count = t->_pending_chain_invalidations.size(); + t->_pending_chain_invalidations_lock.unlock(); + return count; + } + + static size_t drainingChainInvalidationCount() { + return ReferenceChainTracker::instance()->_draining_chain_invalidations.size(); + } + + static size_t pendingChainInvalidationCapacity() { + return ReferenceChainTracker::instance()->_pending_chain_invalidations.capacity(); + } + + static size_t drainingChainInvalidationCapacity() { + return ReferenceChainTracker::instance()->_draining_chain_invalidations.capacity(); + } + + static void drainPendingChainInvalidations() { + ReferenceChainTracker::instance()->drainPendingChainInvalidations(); + } + // Target-selection bridging step: read-only peeks into the resolved-chain cache, for asserting // exactly which klass a chain was resolved for and the tag it was reconstructed from - see // PollWatchedTargetsTest below. @@ -407,8 +446,19 @@ class ReferenceChainsTestAccessor { // leak-correlation tests below. static void recordDiscoveredInstanceForTest(u32 klass_id, jlong tag, bool leak_correlated) { - ReferenceChainTracker::instance()->recordDiscoveredInstance(klass_id, tag, - leak_correlated); + ReferenceChainTracker::instance()->recordDiscoveredInstance( + klass_id, tag, leak_correlated, + ReferenceChainTracker::CacheInvalidationMode::SYNCHRONOUS, + nullptr, 0); + } + + static void recordDiscoveredInstanceDeferredForTest( + u32 klass_id, jlong tag, bool leak_correlated, + const std::vector &snapshot) { + ReferenceChainTracker::instance()->recordDiscoveredInstance( + klass_id, tag, leak_correlated, + ReferenceChainTracker::CacheInvalidationMode::DEFERRED, + snapshot.data(), (int)snapshot.size()); } // Drive restartSearch() directly (the accessor base already set _tags_released, so its assert @@ -955,6 +1005,22 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance() ->seedLeakAccumulationForNewlyWatchedKlass(klass_id); } + + static void pesClear() { + ReferenceChainTracker::instance()->_priority_expand_set.clear(); + } + + static bool pesContains(jlong tag) { + return ReferenceChainTracker::instance()->_priority_expand_set.contains(tag); + } + + static bool pesInsert(jlong tag) { + return ReferenceChainTracker::instance()->_priority_expand_set.insert(tag); + } + + static void pesRebuildFrom(const std::deque &queue) { + ReferenceChainTracker::instance()->_priority_expand_set.rebuildFrom(queue); + } }; #endif // REFERENCE_CHAINS_TEST_ACCESSORS_H diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc index cd7713c13d..3fdecb8a36 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -546,6 +546,78 @@ TEST_F(ResolvedChainCacheTest, MultipleKlassesAllSnapshotTogether) { EXPECT_EQ((std::set{1, 2, 3}), tags); } +TEST_F(ResolvedChainCacheTest, DeferredInvalidationFiltersDeduplicatesAndDrainsExactTags) { + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(1), 1, 0); + ReferenceChainsTestAccessor::cacheChain(2, makeEvent(2), 2, 0); + ReferenceChainsTestAccessor::cacheChain(3, makeEvent(3), 3, 0); + std::vector snapshot = + ReferenceChainsTestAccessor::snapshotResolvedChainTags(); + ASSERT_EQ(3u, snapshot.size()); + + ReferenceChainsTestAccessor::deferResolvedChainInvalidation(2, snapshot); + ReferenceChainsTestAccessor::deferResolvedChainInvalidation(2, snapshot); + ReferenceChainsTestAccessor::deferResolvedChainInvalidation(999, snapshot); + EXPECT_EQ(1u, + ReferenceChainsTestAccessor::pendingChainInvalidationCount()); + + std::vector dump; + ReferenceChainsTestAccessor::drain(&dump); + EXPECT_FALSE(ReferenceChainsTestAccessor::hasResolvedChainForTag(2)); + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(1)); + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(3)); + ASSERT_EQ(2u, dump.size()); + EXPECT_EQ(0u, + ReferenceChainsTestAccessor::pendingChainInvalidationCount()); + EXPECT_EQ(0u, + ReferenceChainsTestAccessor::drainingChainInvalidationCount()); +} + +TEST_F(ResolvedChainCacheTest, DeferredInvalidationIsBoundedByCacheAndKeepsCapacity) { + const int cap = ReferenceChainsTestAccessor::maxResolvedChains(); + for (int i = 0; i < cap; i++) { + ReferenceChainsTestAccessor::cacheChain(i + 1, makeEvent(i + 1), + i + 1, 0); + } + std::vector snapshot = + ReferenceChainsTestAccessor::snapshotResolvedChainTags(); + ASSERT_EQ((size_t)cap, snapshot.size()); + for (jlong tag : snapshot) { + ReferenceChainsTestAccessor::deferResolvedChainInvalidation(tag, + snapshot); + } + for (jlong tag : snapshot) { + ReferenceChainsTestAccessor::deferResolvedChainInvalidation(tag, + snapshot); + } + EXPECT_EQ((size_t)cap, + ReferenceChainsTestAccessor::pendingChainInvalidationCount()); + + ReferenceChainsTestAccessor::drainPendingChainInvalidations(); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_GE(ReferenceChainsTestAccessor::pendingChainInvalidationCapacity(), + (size_t)cap); + EXPECT_GE(ReferenceChainsTestAccessor::drainingChainInvalidationCapacity(), + (size_t)cap); +} + +TEST_F(ResolvedChainCacheTest, RestartDropsPendingTagsBeforeNumericTagReuse) { + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(1), 1, 0); + std::vector snapshot = + ReferenceChainsTestAccessor::snapshotResolvedChainTags(); + ReferenceChainsTestAccessor::deferResolvedChainInvalidation(1, snapshot); + ASSERT_EQ(1u, + ReferenceChainsTestAccessor::pendingChainInvalidationCount()); + + ReferenceChainsTestAccessor::restartSearchForTest(); + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(101), 1, 1); + ReferenceChainsTestAccessor::drainPendingChainInvalidations(); + EXPECT_TRUE(ReferenceChainsTestAccessor::hasResolvedChainForTag(1)); + std::vector dump; + ReferenceChainsTestAccessor::drain(&dump); + ASSERT_EQ(1u, dump.size()); + EXPECT_EQ(101u, dump[0]._target_tag); +} + // A brand-new klass arriving with the cache already at MAX_RESOLVED_CHAINS is dropped (and counted // via REFERENCE_CHAIN_EVENTS_DROPPED, this codebase's own "dropped-event-without-counter" review // lens) rather than evicting some other still-live sample's chain - but refreshing a klass that is diff --git a/ddprof-lib/src/test/cpp/referenceChains_ut.cpp b/ddprof-lib/src/test/cpp/referenceChains_ut.cpp index 88559f365f..ebcae873a1 100644 --- a/ddprof-lib/src/test/cpp/referenceChains_ut.cpp +++ b/ddprof-lib/src/test/cpp/referenceChains_ut.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -55,3 +56,4 @@ static ReferenceChainsGlobalSetup global_setup; #include "referenceChainsTraversalTests.inc" #include "referenceChainsAnchorTests.inc" #include "referenceChainsEventTests.inc" +#include "referenceChainsPesTests.inc"