From a4b0fe64c314cbcda9877938ccb5f0ec68fb9354 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 12 Oct 2026 10:00:53 +0200 Subject: [PATCH 01/28] Java integration tests for reference chains End-to-end leak scenarios (thread-local leaks incl. late-thread and static-field-rooted leaks), leak-tag correlation across JFR round-trips, external-process launches, the search-gate behavior, reference-chain JFR parser tests, and the JavaProfiler test-seam widening they need. --- .../com/datadoghq/profiler/JavaProfiler.java | 183 +++- .../profiler/AbstractProcessProfilerTest.java | 13 +- .../profiler/AbstractProfilerTest.java | 7 + .../datadoghq/profiler/ExternalLauncher.java | 109 ++ .../AggressiveLeakReferenceChainTest.java | 156 +++ .../ExternalProcessReferenceChainTest.java | 208 ++++ .../LeakTagCorrelationReferenceChainTest.java | 239 +++++ .../LeakTagCorrelationScenario.java | 510 +++++++++ .../referencechains/LeakingCacheScenario.java | 353 +++++++ .../ReferenceChainAssertions.java | 216 ++++ .../ReferenceChainJfrParserTest.java | 192 ++++ .../ReferenceChainTestSeamsTest.java | 220 ++++ .../ReferenceChainTrackingTest.java | 979 ++++++++++++++++++ .../StaticFieldGrowingCollectionScenario.java | 278 +++++ .../ThreadLocalLeakReferenceChainTest.java | 226 ++++ .../ThreadLocalLeakScenario.java | 400 +++++++ 16 files changed, 4234 insertions(+), 55 deletions(-) create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 15d7601f9f..6f5bd6374b 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -463,10 +463,15 @@ public Map getDebugCounters() { private static native int getTid0(); /** - * Returns the calling thread's profiler tid. Tests seeding per-tid trends - * via {@link #seedTidTrendSample0} must call this on the leaking thread. - */ - static int getTid() { + * Test seam (debug native builds only): returns the calling thread's profiler tid + * (ProfiledThread::currentTid()'s value on the native side - the same tid space + * {@code seedTidTrendSample0} matches tracked instances' allocating threads + * against). Scenarios seeding per-(klass, tid) trend ramps must call this ON + * the leaking thread and pass the result to {@code seedTidTrendSample0}, or + * the qualifying tid will match no tracked instance and no leak tag will + * ever be assigned to the scenario's real instances. + */ + public static int getTid() { return getTid0(); } @@ -539,109 +544,179 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l public static native boolean testTlsPrimingAvailable(); /** - * Test seam. The test seams below are implemented in debug native builds only; - * in release builds they are no-ops. + * Test seam (debug native builds only - a no-op returning {@code false}/{@code 0}/an + * empty array in release builds): decouples LivenessTracker's leak-candidate + * detection from ReferenceChainTracker's chain reconstruction, each independently + * verifiable end-to-end without depending on both the probabilistic JVMTI heap + * sampler and the reference-chain BFS search organically producing the right + * conditions in the same test run. *

- * Enables/disables per-klass population tracking without a live-JVM - * {@code initialize()}. Returns {@code true} on debug builds. + * Enables/disables LivenessTracker's per-klass population tracking directly, + * bypassing {@code initialize()}'s live-JVM requirement. Returns {@code true} on + * debug builds. */ - static native boolean setGcGenerationsEnabled0(boolean enabled); + public static native boolean setGcGenerationsEnabled0(boolean enabled); /** - * Test seam: seeds one epoch of population history for {@code klassId}. + * Test seam (debug native builds only): seeds one epoch's worth of population + * history for {@code klassId} directly into LivenessTracker's ring buffer, + * bypassing real allocation sampling. Repeated calls (with distinct + * {@code epoch} values) build up a trend {@link #selectLeakCandidateKlassIds0()} + * can then rank, letting a test assert a slope signal would be generated for a + * chosen klass id without waiting on real GC epochs. */ - static native void seedKlassPopulationSample0(int klassId, int count, long epoch); + public static native void seedKlassPopulationSample0(int klassId, int count, long epoch); /** - * Test seam: seeds one epoch of per-tid trend history for {@code klassId}. - * {@code tid} should come from {@link #getTid()}. + * Test seam (debug native builds only): seeds one epoch's worth of per-tid trend + * history for {@code klassId}, the (klass, tid) qualification + * selectLeakCandidates() now requires on top of the klass-level ramp seeded by + * {@link #seedKlassPopulationSample0}. Repeated calls with the same rising shape + * (and the same {@code tid}) build the sustained rise that qualifies {@code tid} + * as a leak-site thread. {@code tid} must be the leaking thread's real profiler + * tid from {@link #getTid()} whenever the scenario also relies on leak tagging + * of real tracked instances; see that method's own comment. */ - static native void seedTidTrendSample0(int klassId, int tid, int count, long epoch); + public static native void seedTidTrendSample0(int klassId, int tid, int count, long epoch); /** - * Test seam: makes {@code representative} the leak-candidate representative - * of {@code klassId}. + * Test seam (debug native builds only): wires {@code representative} in as {@code klassId}'s + * leak-candidate representative directly (a fresh weak global ref owned by LivenessTracker), + * bypassing the real allocation-sampling path that would otherwise populate this. Combined + * with {@link #seedKlassPopulationSample0} and {@link #tagAsReferenceChainRoot0}, lets a test + * join a synthetic slope signal to a real, directly-tagged object so + * {@link #pollReferenceChainTargets0()}'s bridging step can be exercised end-to-end with + * neither the real sampler nor the real root-seeded walk involved. */ - static native void setKlassPopulationRepresentativeForTest0(int klassId, Object representative); + public static native void setKlassPopulationRepresentativeForTest0(int klassId, Object representative); /** - * Test seam: clears the per-klass population table. + * Test seam (debug native builds only): clears LivenessTracker's per-klass population table, + * so a later test in the same JVM does not observe leak candidates seeded by an earlier one. */ - static native void resetKlassPopulationForTest0(); + public static native void resetKlassPopulationForTest0(); /** - * Test seam: the klass ids currently selected as leak candidates (top 5). + * Test seam (debug native builds only): returns the klass ids LivenessTracker's + * real leak-candidate ranking (positive population slope, top 5) currently + * selects - the same call ReferenceChainTracker's restart gate and target-polling + * bridge use in production, exposed here so a test can assert a slope signal was + * generated (real or seeded via {@link #seedKlassPopulationSample0}) without + * needing a reference-chain search to also be running. */ - static native int[] selectLeakCandidateKlassIds0(); + public static native int[] selectLeakCandidateKlassIds0(); /** - * Test seam: tags {@code target} as a reference-chain frontier root. - * Returns the frontier tag, or {@code 0} on failure. + * Test seam (debug native builds only): tags {@code target} and inserts it + * directly as a reference-chain frontier root, bypassing ReferenceChainTracker's + * normal discovery path (a root-seeded FollowReferences walk) and + * LivenessTracker's leak-candidate selection entirely. Lets a test drive + * {@link #runReferenceChainPass0()}/{@link #pollReferenceChainTargets0()} against + * a known, caller-chosen live object. Returns the assigned frontier tag (matching + * the {@code target_tag} a resulting {@code datadog.ReferenceChain} event + * reports), or {@code 0} on failure (reference chains disabled, or the frontier + * table is at capacity). */ - static native long tagAsReferenceChainRoot0(Object target); + public static native long tagAsReferenceChainRoot0(Object target); /** - * Test seam: runs one reference-chain search pass synchronously. Returns - * {@code false} if reference chains are disabled or not started. + * Test seam (debug native builds only): runs exactly one bounded BFS pass of the + * reference-chain search synchronously, rather than waiting on the tracker's own + * background thread/cadence. Returns {@code false} if reference chains are + * disabled or the tracker was never started. */ - static native boolean runReferenceChainPass0(); + public static native boolean runReferenceChainPass0(); /** - * Test seam: runs one poll of the leak-candidate to chain bridge synchronously. + * Test seam (debug native builds only): runs one poll of + * ReferenceChainTracker's LivenessTracker-to-chain-reconstruction bridging step + * synchronously - for each current leak candidate already discovered by a prior + * {@link #runReferenceChainPass0()} walk, reconstructs and queues its chain + * event, rather than waiting on the background thread's own scheduling cycle. */ - static native void pollReferenceChainTargets0(); + public static native void pollReferenceChainTargets0(); /** - * Test seam: drains queued reference-chain events and returns their count. + * Test seam (debug native builds only): drains and returns the number of + * reference-chain events queued by {@link #pollReferenceChainTargets0()} so far + * (the same queue {@code Profiler.dump()} drains in production to write + * {@code datadog.ReferenceChain} JFR events) - lets a test assert a chain was + * actually reconstructed without needing a real JFR dump. */ - static native int drainReferenceChainEventCount0(); + public static native int drainReferenceChainEventCount0(); /** - * Test seam: resets the search state and releases held tags, so the next - * pass walks from the roots again. + * Test seam (debug native builds only): resets ReferenceChainTracker's search/frontier state + * back to a brand-new tracker's, releasing any tags a previous search still held. Since the + * tracker is a process-wide singleton, an in-process test that needs its own genuine first + * root-seeded walk (runPass() only re-walks from the roots once per search's whole lifetime) + * calls this at the start of its test body to force one, rather than depending on being the + * first reference-chain test to run in a shared test JVM. */ - static native void resetReferenceChainSearchForTest0(); + public static native void resetReferenceChainSearchForTest0(); /** - * Test seam: position of {@code target} in the pending-expansion queue - * ({@code 0} = next), {@code -1} if already expanded, {@code -2} if never - * admitted. Does not tag {@code target}. + * Test seam (debug native builds only): diagnostic-only, does not tag {@code target}. Reads + * target's existing JVMTI tag (0 if the real search has never admitted it) and reports its + * FIFO distance from the front of ReferenceChainTracker's pending-expansion queue: {@code >=0} + * (0 = expands next) if still queued, {@code -1} if tagged but no longer queued (already + * expanded), or {@code -2} if never admitted at all. */ - static native long getReferenceChainPendingPositionForTest0(Object target); + public static native long getReferenceChainPendingPositionForTest0(Object target); /** - * Test seam: size of the pending-expansion queue. + * Test seam (debug native builds only): the current size of ReferenceChainTracker's + * pending-expansion queue, for computing {@link #getReferenceChainPendingPositionForTest0}'s + * position as a fraction of the current backlog. */ - static native long getReferenceChainPendingSizeForTest0(); + public static native long getReferenceChainPendingSizeForTest0(); /** - * Test seam: seeds one heap-floor sample for the time-to-OOM projection. - * Timestamps only need to be increasing. + * Test seam (debug native builds only): seeds one heap-floor-ring sample - the input to + * {@code LivenessTracker::secondsToOOM()}'s time-to-OOM projection - directly, bypassing the + * real {@code GarbageCollectionFinish} callback. {@code timestampNs} values are only ever + * compared against each other, never against a real wall clock, so a test may use any + * self-consistent, strictly increasing sequence to build an arbitrary rising or flat + * heap-usage-over-time history without waiting on real GCs. */ - static native void heapFloorRecordForTest0(long usedBytes, long timestampNs); + public static native void heapFloorRecordForTest0(long usedBytes, long timestampNs); /** - * Test seam: overrides the max heap size used by the time-to-OOM projection. + * Test seam (debug native builds only): overrides the max-heap-size {@code secondsToOOM()} + * projects against, bypassing the real {@code Runtime.maxMemory()} resolution - so a test can + * exercise the projection deterministically, independent of whatever {@code -Xmx} this JVM's + * own shared, no-{@code forkEvery} fork happens to run with. */ - static native void setMaxHeapBytesForTest0(long maxHeapBytes); + public static native void setMaxHeapBytesForTest0(long maxHeapBytes); /** - * Test seam: {@code false} stops GCs from recording heap-floor samples, - * {@code true} restores it. + * Test seam (debug native builds only): temporarily disables {@code onGC()}'s own + * {@code recordHeapFloorSample()} call so a test can seed the heap-floor ring exclusively + * via {@link #heapFloorRecordForTest0(long, long)} without a real GC interleaving a sample + * with a real {@code OS::nanotime()} timestamp and real heap usage, corrupting + * {@code secondsToOOM()}'s projection. Pass {@code false} to disable, {@code true} to restore. */ - static native void setHeapFloorRecordingForTest0(boolean enabled); + public static native void setHeapFloorRecordingForTest0(boolean enabled); /** - * Test seam: evaluates the real search scheduling gate without executing a BFS pass. - * This is not a read-only query: under the native engine lock it may account a completed - * search's pain and restart/reset terminal search state. + * Test seam (debug native builds only): reports whether ReferenceChainTracker's search-restart + * gate ({@code canAffordNewSearch()} -> {@code hasLeakSignal()}) would currently allow a + * fresh/terminal search to start - in particular, whether {@code secondsToOOM()}'s urgent-OOM + * bypass opens this gate even with zero per-klass leak candidate (confirmable in the same test + * via {@link #selectLeakCandidateKlassIds0()}). Unlike {@link #runReferenceChainPass0()}, which + * calls {@code runPass()} unconditionally, this reads the gate itself without running a pass. */ - static native boolean shouldRunPassForTest0(); + public static native boolean shouldRunPassForTest0(); /** - * Test seam: number of passes run for the current search. + * Test seam (debug native builds only): the number of BFS passes run for the current/most + * recent reference-chain search. Lets a test note this count before creating an object, then + * wait for it to advance before trusting a match against that object - the only way to be + * certain the match came from a pass whose own {@code expandFrontier()} (and therefore {@code + * collectStaleExpandedEntriesForRotation()}) ran strictly after the object existed, rather + * than from the same pass racing the object's creation. */ - static native int referenceChainPassesRunForTest0(); + public static native int referenceChainPassesRunForTest0(); // ---- Test-only reads of the current thread's OTEP record ---------------------------------- // Each resolves the current carrier's record directly (like the write primitives above) with diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java index 6c0bc0685f..e213b9a005 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java @@ -36,6 +36,17 @@ protected final LaunchResult launch(String target, List jvmArgs, String } protected final LaunchResult launch(String target, List jvmArgs, String commands, Map env, Function onStdoutLine, Function onStderrLine) throws Exception { + return launch(target, jvmArgs, commands, env, 10, onStdoutLine, onStderrLine); + } + + /** + * Same as the 6-arg overload above, but with a caller-supplied wait timeout instead of the + * hardcoded 10 seconds - needed by scenarios that run substantially longer than a plain + * attach/init check, e.g. {@code ExternalProcessReferenceChainTest}'s population-growth loop + * (up to 25 rounds plus a grace period, ~20s+ observed in-process, plus a whole separate + * JVM's own startup/classloading cost on top). + */ + protected final LaunchResult launch(String target, List jvmArgs, String commands, Map env, long timeoutSeconds, Function onStdoutLine, Function onStderrLine) throws Exception { String javaHome = System.getenv("JAVA_TEST_HOME"); if (javaHome == null) { javaHome = System.getenv("JAVA_HOME"); @@ -130,7 +141,7 @@ protected final LaunchResult launch(String target, List jvmArgs, String stdoutReader.start(); stderrReader.start(); - boolean val = p.waitFor(10, TimeUnit.SECONDS); + boolean val = p.waitFor(timeoutSeconds, TimeUnit.SECONDS); if (!val) { p.destroyForcibly(); p.waitFor(5, TimeUnit.SECONDS); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java index b059bb655c..a6b83f9b04 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java @@ -73,6 +73,12 @@ public static double scaledSize(JfrEvent item) { protected JavaProfiler profiler; private Path jfrDump; + // Set at the very start of setupProfiler(), before getProfilerCommand() is + // consulted, so a subclass can branch its command on the current test + // method (e.g. distinct hop/budget/frontier-cap values per @Test) without + // needing separate test classes per configuration. + protected TestInfo testInfo; + private Duration cpuInterval; private Duration wallInterval; @@ -200,6 +206,7 @@ protected void withTestAssumptions() {} @BeforeEach public void setupProfiler(TestInfo testInfo) throws Exception { + this.testInfo = testInfo; Assumptions.assumeTrue(isPlatformSupported()); withTestAssumptions(); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java index 2dbb429668..dd18a91a58 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java @@ -5,10 +5,16 @@ package com.datadoghq.profiler; +import com.datadoghq.profiler.referencechains.LeakTagCorrelationScenario; +import com.datadoghq.profiler.referencechains.LeakingCacheScenario; +import com.datadoghq.profiler.referencechains.StaticFieldGrowingCollectionScenario; +import com.datadoghq.profiler.referencechains.ThreadLocalLeakScenario; + import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; import java.lang.reflect.Method; +import java.nio.file.Paths; import java.util.Random; import java.util.concurrent.atomic.LongAdder; @@ -30,6 +36,26 @@ * CPU concurrently on the main thread, on a plain {@code new Thread(Runnable)} and on a * two-level {@link Thread} subclass, and stops the profiler again. The resulting recording * holds samples rooted at each of the three thread entry points; see {@code EntryFrameTest} + *

  • leak-cache "<start command>|||<scratch dump path>" - starts the profiler with + * the given start command, then runs {@link LeakingCacheScenario} in this process and exits + * directly (no readiness/stdin-signal handshake, unlike the other modes above) once it + * prints its one result line. Used by {@code ExternalProcessReferenceChainTest} + * (referencechains package) to prove the reference-chains mechanism end-to-end against a + * genuinely separate JVM, not the in-process dynamic-attach lifecycle every other test in + * this module uses - see that scenario's own class comment for why a *separate* process is + * load-bearing here (the one-shot root-seeded BFS walk is a process-wide, once-ever + * resource).
    • + *
  • leak-static-field "<start command>|||<scratch dump path>" - same packing and + * lifecycle as {@code leak-cache} above, but runs {@link StaticFieldGrowingCollectionScenario} + * instead: a {@code static final List} field appended to (never reassigned) after + * its owning class has already been swept once by {@code admitStaticFieldRoots()} - + * mirroring the real leak generator found in the {@code prof-analyzer-hotdog-jb} pod.
    • + *
  • leak-correlation "<start command>|||<scratch dump path>" - same packing and + * lifecycle as {@code leak-cache} above, but runs {@link LeakTagCorrelationScenario} + * instead: a static-collection leak coexisting with ephemeral stack-local noise of another + * class and a large live filler graph, asserting the emitted chain's targetTag is a + * leak-tag-pool tag matching a HeapLiveObject's leakTag - the chain-to-live-heap-sample + * correlation use case.
    • * */ public class ExternalLauncher { @@ -201,6 +227,89 @@ public static void main(String[] args) throws Exception { worker.start(); } } + } else if (args[0].equals("leak-cache")) { + // "|||" packed into one args[1] string rather + // than extending AbstractProcessProfilerTest.launch()'s generic 2-arg + // (target, commands) contract with a 3rd argument every other mode would have to + // ignore. + String packed = args.length == 2 ? args[1] : ""; + int sep = packed.indexOf("|||"); + if (sep < 0) { + throw new IllegalArgumentException( + "leak-cache requires \"|||\", got: " + packed); + } + String commands = packed.substring(0, sep); + String scratchPath = packed.substring(sep + "|||".length()); + JavaProfiler instance = JavaProfiler.getInstance(); + // LeakingCacheScenario.run() itself calls instance.execute(commands), not here - + // it needs to seed its cache fixture *before* starting the profiler (see that + // method's own comment for why). + LeakingCacheScenario.run(instance, commands, Paths.get(scratchPath)); + System.out.flush(); + // Deliberately exits here rather than falling through to the shared + // "[ready]" + stdin-signal handshake below: this mode runs to completion in one + // shot (no live back-and-forth with the parent needed) and its own result line + // has already been printed by LeakingCacheScenario.run(). + System.exit(0); + } else if (args[0].equals("leak-static-field")) { + // Same "|||" packing as leak-cache above. + String packed = args.length == 2 ? args[1] : ""; + int sep = packed.indexOf("|||"); + if (sep < 0) { + throw new IllegalArgumentException( + "leak-static-field requires \"|||\", got: " + packed); + } + String commands = packed.substring(0, sep); + String scratchPath = packed.substring(sep + "|||".length()); + JavaProfiler instance = JavaProfiler.getInstance(); + StaticFieldGrowingCollectionScenario.run(instance, commands, Paths.get(scratchPath)); + System.out.flush(); + System.exit(0); + } else if (args[0].equals("leak-correlation")) { + // Same "|||" packing as leak-cache above. + String packed = args.length == 2 ? args[1] : ""; + int sep = packed.indexOf("|||"); + if (sep < 0) { + throw new IllegalArgumentException( + "leak-correlation requires \"|||\", got: " + packed); + } + String commands = packed.substring(0, sep); + String scratchPath = packed.substring(sep + "|||".length()); + JavaProfiler instance = JavaProfiler.getInstance(); + LeakTagCorrelationScenario.run(instance, commands, Paths.get(scratchPath)); + System.out.flush(); + System.exit(0); + } else if (args[0].equals("threadlocal-leak")) { + // Same "|||" packing as leak-cache above. + String packed = args.length == 2 ? args[1] : ""; + int sep = packed.indexOf("|||"); + if (sep < 0) { + throw new IllegalArgumentException( + "threadlocal-leak requires \"|||\", got: " + packed); + } + String commands = packed.substring(0, sep); + String scratchPath = packed.substring(sep + "|||".length()); + JavaProfiler instance = JavaProfiler.getInstance(); + ThreadLocalLeakScenario.run(instance, commands, Paths.get(scratchPath)); + System.out.flush(); + System.exit(0); + } else if (args[0].equals("threadlocal-leak-late")) { + // Same packing as threadlocal-leak, but the leak thread is + // started only AFTER the recording began - the regression + // shape for Profiler::onThreadStart's registerThreadObject() + // wiring (ThreadLocalLeakScenario.run()'s lateThread comment). + String packed = args.length == 2 ? args[1] : ""; + int sep = packed.indexOf("|||"); + if (sep < 0) { + throw new IllegalArgumentException( + "threadlocal-leak-late requires \"|||\", got: " + packed); + } + String commands = packed.substring(0, sep); + String scratchPath = packed.substring(sep + "|||".length()); + JavaProfiler instance = JavaProfiler.getInstance(); + ThreadLocalLeakScenario.run(instance, commands, Paths.get(scratchPath), true); + System.out.flush(); + System.exit(0); } } finally { System.out.println("[ready]"); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java new file mode 100644 index 0000000000..8cc7f9a440 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java @@ -0,0 +1,156 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * E2E coverage for {@code LivenessTracker::secondsToOOM()}'s urgent-OOM bypass + * ({@code ReferenceChainTracker::hasLeakSignal()}, referenceChains.cpp). Complements + * {@link ReferenceChainTrackingTest}'s organic coverage of a slow leak - a gradually-growing + * population that clears {@code selectLeakCandidates()}'s per-klass hysteresis gate over many + * rounds - with the opposite, previously-unexercised case: a heap-wide leak growing fast enough + * that a real deployment would OOM before any single klass's own population history ever clears + * that same hysteresis gate. + * + *

    Reproducing a genuine near-OOM heap in this shared, no-{@code forkEvery} test JVM is + * impractical (see {@link ReferenceChainTrackingTest}'s own header comment on the JVM this class + * shares with every other {@code ddprof-test} class). The debug-only + * {@code heapFloorRecordForTest0()}/{@code setMaxHeapBytesForTest0()} seams (javaApi.cpp) instead + * seed {@code LivenessTracker}'s aggregate heap-floor-ring history and its max-heap-bytes cache + * directly, so {@code secondsToOOM()}'s projection can be driven under + * {@code OOM_URGENT_THRESHOLD_S} (referenceChains.h) deterministically, decoupled from this JVM's + * real {@code -Xmx} and from real GC timing. {@code shouldRunPassForTest0()} then reads + * {@code ReferenceChainTracker::shouldRunPass()} - the search-restart gate {@code hasLeakSignal()} + * feeds - directly, rather than running a real pass, so this class never depends on + * {@code selectLeakCandidates()} ranking anything, a real JVMTI heap walk, or a JFR event. + */ +public class AggressiveLeakReferenceChainTest extends AbstractProfilerTest { + + @Override + protected String getProfilerCommand() { + // generations=true: gates LivenessTracker::gcGenerationsEnabled() - hasLeakSignal() (which + // shouldRunPassForTest0() ultimately reads) short-circuits to true immediately when this is + // unset (its own first check, referenceChains.cpp), which would make every test below + // trivially pass regardless of the urgent-OOM bypass this class exists to exercise. + // referencechains=true: constructs the FrontierTable/ReferenceChainTracker singleton at all - + // shouldRunPassForTest0() has nothing to read otherwise. + return "generations=true,referencechains=true:hops=32:budget=500:ttl=60000:framecap=2000000"; + } + + @Override + protected boolean isPlatformSupported() { + // Mirrors ReferenceChainTrackingTest's own guard - FollowReferences/tag-based frontier walking + // assumes a HotSpot-shaped JVMTI heap implementation. + return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); + } + + private static void assumeDebugBuild() { + assumeTrue("debug".equals(System.getProperty("ddprof_test.config")), + "heapFloorRecordForTest0/setMaxHeapBytesForTest0/shouldRunPassForTest0 only exist in a " + + "debug native build (javaApi.cpp's #ifdef DEBUG guard)"); + } + + /** + * Seeds ten heap-floor-ring samples rising fast enough that {@code secondsToOOM()}'s projection + * lands well under {@code OOM_URGENT_THRESHOLD_S} (300s), with LivenessTracker's per-klass + * population table left empty throughout - {@code selectLeakCandidateKlassIds0()} returns + * nothing at any point in this test. Then waits for the BFS thread to actually run passes, + * proving the search-restart gate opened from the urgent-OOM projection alone. + * + *

    The gate is asserted through the live BFS thread rather than by calling + * {@code shouldRunPassForTest0()} directly: opening the gate spends the episode's one-shot + * urgency entitlement ({@code _urgent_search_spent}) on the FIRST evaluation, and the BFS + * thread - whose cadence ramps down to ~10ms once urgency latches - races the test's own + * call for that first evaluation. A direct call therefore only returns true when it wins + * that race, which made this test flaky by construction. A pass only ever runs after + * {@code shouldRunPass()} returned true, and with {@code generations=true} plus a provably + * empty per-klass population table the urgent-OOM projection is the only possible trigger - + * so "passes ran while zero candidates existed" is the same assertion, without the race. + */ + @Test + public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() + throws InterruptedException { + assumeDebugBuild(); + JavaProfiler.setHeapFloorRecordingForTest0(false); + JavaProfiler.resetKlassPopulationForTest0(); + JavaProfiler.resetReferenceChainSearchForTest0(); + try { + // 1000 MiB fake max heap, rising from 100 MiB to 991 MiB over 9 (fake) seconds: ~99 MiB/s, + // 9 MiB of headroom left at the last sample -> projected time-to-OOM ~= 9/99 s, far under + // OOM_URGENT_THRESHOLD_S = 300s. + JavaProfiler.setMaxHeapBytesForTest0(1000L * 1024 * 1024); + for (int i = 0; i < 10; i++) { + long usedBytes = (100L + i * 99L) * 1024 * 1024; + long timestampNs = i * 1_000_000_000L; + JavaProfiler.heapFloorRecordForTest0(usedBytes, timestampNs); + } + + int[] candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + assertTrue(candidates == null || candidates.length == 0, + "This test's own precondition: no per-klass candidate should exist, so passes running " + + "below can only come from the aggregate urgent-OOM bypass"); + + int passesBefore = JavaProfiler.referenceChainPassesRunForTest0(); + // Idle BFS cadence is ~1s/pass; urgency ramps it to ~10ms after the latch. 20s covers the + // first evaluation landing up to one idle cadence after the seeding loop. + long deadline = System.currentTimeMillis() + 20_000; + while (JavaProfiler.referenceChainPassesRunForTest0() == passesBefore + && System.currentTimeMillis() < deadline) { + Thread.sleep(50); + } + assertTrue(JavaProfiler.referenceChainPassesRunForTest0() > passesBefore, + "Expected the BFS thread to run passes from the urgent heap-wide OOM projection alone, " + + "with zero per-klass leak candidate"); + + candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + assertTrue(candidates == null || candidates.length == 0, + "The passes that just ran must still have had zero per-klass candidates - the " + + "aggregate urgent-OOM projection was the only possible trigger"); + } finally { + JavaProfiler.setMaxHeapBytesForTest0(-1); + JavaProfiler.setHeapFloorRecordingForTest0(true); + } + } + + /** + * Same empty per-klass population table as above, but a flat heap floor (no growth at all) - + * {@code secondsToOOM()} has nothing to project from, so this test's own gate must stay closed. + * Without this control, a stubbed-out {@code secondsToOOM()} that always signals urgency would + * pass the test above just as well. + */ + @Test + public void shouldNotOpenSearchGateOnFlatHeapFloorWithNoLeakCandidate() { + assumeDebugBuild(); + JavaProfiler.setHeapFloorRecordingForTest0(false); + JavaProfiler.resetKlassPopulationForTest0(); + JavaProfiler.resetReferenceChainSearchForTest0(); + try { + JavaProfiler.setMaxHeapBytesForTest0(1000L * 1024 * 1024); + for (int i = 0; i < 10; i++) { + JavaProfiler.heapFloorRecordForTest0(200L * 1024 * 1024, i * 1_000_000_000L); + } + + int[] candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + assertTrue(candidates == null || candidates.length == 0, + "This test's own precondition: no per-klass candidate should exist either"); + + assertFalse(JavaProfiler.shouldRunPassForTest0(), + "Expected the search-restart gate to stay closed: no heap growth and no per-klass " + + "candidate give hasLeakSignal() nothing to trust"); + } finally { + JavaProfiler.setMaxHeapBytesForTest0(-1); + JavaProfiler.setHeapFloorRecordingForTest0(true); + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java new file mode 100644 index 0000000000..cb91ad34f8 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -0,0 +1,208 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProcessProfilerTest; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +/** + * Genuinely separate-process end-to-end coverage for the reference-chains target-selection + * mechanism, complementing {@link ReferenceChainTrackingTest}'s in-process coverage: runs a real + * "leaking Java app" (an ever-growing, never-evicted {@code HashMap}-backed cache - see + * {@link LeakingCacheScenario}) in a genuinely separate child JVM, launched the same way + * {@code JavaProfilerTest}/{@code JVMAccessTest} already launch child processes, and asserts on + * that child's own reported result rather than reading a JFR file back out of this test's own + * process. + * + *

    Why a separate process, not another {@code @Test} in {@code ReferenceChainTrackingTest}: + * an earlier attempt added a second success-path {@code @Test} method there, using this exact + * same {@code HashMap}-based leak shape, sharing that class's in-process {@code AbstractProfilerTest} + * dynamic-attach lifecycle. It failed reliably whenever it ran after + * {@code ReferenceChainTrackingTest}'s own {@code ChainLink} test in the same JVM - not a + * test-ordering bug, but a genuine, non-obvious property of {@code ReferenceChainTracker}: + * {@code runPass()} (referenceChains.cpp) performs exactly one root-seeded {@code FollowReferences} + * walk per search's *entire lifetime* ({@code _search_started}); every later pass only expands + * frontier entries that walk already discovered ({@code expandFrontier()}) - it never + * re-examines GC roots or revisits an already-{@code EXPANDED} entry for newly-added children. + * Since {@code ReferenceChainTracker} is a process-wide singleton, only the *first* test to ever + * call {@code runPass()} in a given JVM gets a real root-seeded walk; every subsequent test's own, + * independently-rooted local variables are structurally invisible to the mechanism afterward, no + * matter how they are built. A genuinely separate child JVM per scenario sidesteps this + * entirely: each process gets its own fresh {@code ReferenceChainTracker} singleton, and + * therefore its own guaranteed first-ever root walk. + */ +@Tag("slow") +public class ExternalProcessReferenceChainTest extends AbstractProcessProfilerTest { + + @Test + void shouldReconstructReferrerChainInSeparateProcess() throws Exception { + // Mirrors ReferenceChainTrackingTest.isPlatformSupported()'s own guard - FollowReferences/ + // tag-based frontier walking assumes a HotSpot-shaped JVMTI heap implementation. + assumeFalse(Platform.isJavaVersion(8)); + assumeFalse(Platform.isJ9()); + assumeFalse(Platform.isZing()); + + Path scratchDumpPath = Files.createTempFile("referencechains-external-process", ".jfr"); + // LeakingCacheScenario.run() (inside the child) creates this file on its own first dump() - + // an empty placeholder here would make that first dump() attempt fail confusingly. + Files.deleteIfExists(scratchDumpPath); + // "start" requires a "jfr,file=..." clause (found the hard way: JavaProfiler.execute() + // throws "Flight Recorder output file is not specified" without one) even though this + // scenario never reads its content - only the explicit dump() calls + // LeakingCacheScenario.run() makes to scratchDumpPath matter. + Path continuousJfrPath = Files.createTempFile("referencechains-external-process-continuous", ".jfr"); + try { + // budget=200000 (up from the in-process test's 4000, found via TEST_LOG instrumentation + // in referenceChains.cpp/livenessTracker.cpp): a genuinely fresh external JVM's + // reachable-from-roots graph (full JUnit/JMC/Gradle-worker classpath, bootstrapped from + // scratch - no benefit from a warm, already-running shared worker JVM) is far larger than + // the in-process test's, and each pass's own JVMTI walk cost scales with cumulative + // frontier size, not with this scenario's own allocation rate - budget=4000 stayed + // truncated=1 at 76000+ admitted edges after 19 real passes and never got close to this + // scenario's own cache before the test's own timeout. _budget is the pacing controller's + // hard ceiling (see updatePacing()'s own comment, referenceChains.cpp) - pausetarget alone + // cannot compensate for a ceiling set too low, only pacing *within* it. + // memory=64:l:1.0 - the explicit 100% live-sample ratio removes the + // default 10% subsampling lottery from the tracked-instance pool + // (see LeakTagCorrelationReferenceChainTest's own comment for the + // full analysis); total volume stays capped by the 256KiB interval + // floor, so the tracking table remains tiny. + String startCommand = "start,memory=64:l:1.0,generations=true," + + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:pausetarget=60000" + + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); + // Packed into one args[1] string - see ExternalLauncher's own "leak-cache" mode comment + // for why (avoids extending AbstractProcessProfilerTest.launch()'s generic (target, + // commands) contract with a 3rd argument every other mode would have to ignore). + String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); + + // Propagates this JVM's own ddprof_test.config (set by ProfilerTestPlugin.kt on the + // ddprof-test Test task itself, not inherited by a ProcessBuilder-launched child on its + // own) so LeakingCacheScenario can use the same debug-only seeded-representative fallback + // ReferenceChainTrackingTest already relies on instead of depending purely on real + // allocation-sampling timing. + List jvmArgs = Collections.singletonList( + "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + + AtomicReference resultLine = new AtomicReference<>(); + LaunchResult result = launch("leak-cache", jvmArgs, packedCommand, + Collections.emptyMap(), + // Generous: a whole separate JVM's startup/classloading cost, on top of the same + // up-to-25-round population-growth loop plus grace period that takes ~20s in-process + // (ReferenceChainTrackingTest's own history). + 90, + line -> { + if (line.startsWith(LeakingCacheScenario.FOUND_MARKER) + || line.equals(LeakingCacheScenario.NOT_FOUND_MARKER) + || line.startsWith(LeakingCacheScenario.NO_HASHMAP_INTERNALS_MARKER) + || line.startsWith(LeakingCacheScenario.CHAIN_NOT_PERSISTED_MARKER)) { + resultLine.set(line); + } + return LineConsumerResult.CONTINUE; + }, + null); + + assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); + assertNotNull(resultLine.get(), + "Child process never printed a recognizable result marker on stdout"); + assertEquals( + LeakingCacheScenario.FOUND_MARKER + LeakingCacheScenario.CachedPayload.class.getName(), + resultLine.get(), + "Expected a successfully reconstructed chain whose leaf is CachedPayload, threading " + + "through java.util.HashMap's own internal storage - got: " + resultLine.get()); + } finally { + Files.deleteIfExists(scratchDumpPath); + Files.deleteIfExists(continuousJfrPath); + } + } + + /** + * Mirrors {@link #shouldReconstructReferrerChainInSeparateProcess()}, but against {@link + * StaticFieldGrowingCollectionScenario} instead of {@link LeakingCacheScenario}: a {@code + * static final List} field appended to (never reassigned) after its owning class has + * already been swept once by {@code admitStaticFieldRoots()} (referenceChains.cpp) - the exact + * shape found in the {@code prof-analyzer-hotdog-jb} pod's real leak generator, as opposed to + * {@code LeakingCacheScenario}'s local-variable (stack-root) cache. Local, fast reproduction of + * whether {@code collectStaleExpandedEntriesForRotation()}'s rotation mechanism actually + * re-expands an already-{@code EXPANDED} static-field-rooted collection to pick up elements + * added after its one-time sweep. + */ + @Test + void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exception { + assumeFalse(Platform.isJavaVersion(8)); + assumeFalse(Platform.isJ9()); + assumeFalse(Platform.isZing()); + + Path scratchDumpPath = Files.createTempFile("referencechains-static-field-external-process", ".jfr"); + Files.deleteIfExists(scratchDumpPath); + Path continuousJfrPath = Files.createTempFile("referencechains-static-field-external-process-continuous", ".jfr"); + try { + // See ExternalProcessReferenceChainTest.shouldReconstructReferrerChainInSeparateProcess()'s + // own comment for why budget/pausetarget are raised this far above the in-process test's + // defaults for a genuinely fresh, cold external JVM. + // memory=64:l:1.0 - the explicit 100% live-sample ratio removes the + // default 10% subsampling lottery from the tracked-instance pool + // (see LeakTagCorrelationReferenceChainTest's own comment for the + // full analysis); total volume stays capped by the 256KiB interval + // floor, so the tracking table remains tiny. + String startCommand = "start,memory=64:l:1.0,generations=true," + + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:pausetarget=60000" + + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); + String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); + + List jvmArgs = Collections.singletonList( + "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + + AtomicReference resultLine = new AtomicReference<>(); + LaunchResult result = launch("leak-static-field", jvmArgs, packedCommand, + Collections.emptyMap(), + // Wider than shouldReconstructReferrerChainInSeparateProcess()'s 90s: this scenario's + // own tail wait (StaticFieldGrowingCollectionScenario.run()) needs up to ~60s on top of + // the round loop for a cold JVM's first full pass to actually complete - see that + // method's own comment. + 150, + line -> { + if (line.startsWith(StaticFieldGrowingCollectionScenario.FOUND_MARKER) + || line.equals(StaticFieldGrowingCollectionScenario.NOT_FOUND_MARKER)) { + resultLine.set(line); + } + return LineConsumerResult.CONTINUE; + }, + null); + + assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); + assertNotNull(resultLine.get(), + "Child process never printed a recognizable result marker on stdout"); + assertEquals( + // "byte[]", not byte[].class.getName()'s "[B": StaticFieldGrowingCollectionScenario + // prints match.chain.get(0).getFullName(), and JMC's IMCType renders array types in + // Java source notation, not JVM signature notation - see + // ReferenceChainAssertions.jmcStyleName()'s own comment for why matching required the + // same distinction. + StaticFieldGrowingCollectionScenario.FOUND_MARKER + "byte[]", + resultLine.get(), + "Expected a successfully reconstructed chain to a byte[] chunk appended to the static " + + "field's list after its one-time admitStaticFieldRoots() sweep - got: " + resultLine.get()); + } finally { + Files.deleteIfExists(scratchDumpPath); + Files.deleteIfExists(continuousJfrPath); + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java new file mode 100644 index 0000000000..656be82fa1 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java @@ -0,0 +1,239 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProcessProfilerTest; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +/** + * End-to-end coverage of the leak-tag correlation use case in a genuinely separate child + * JVM (same rationale as {@code ExternalProcessReferenceChainTest}'s own class comment - + * the one-shot root-seeded BFS walk is a per-process resource): a growing static-collection + * leak amid ephemeral stack-local noise and a large live filler graph must be reported as a + * {@code datadog.ReferenceChain} whose {@code targetTag} is in the leak-tag pool's range + * AND matches a {@code datadog.HeapLiveObject}'s {@code leakTag}. See + * {@link LeakTagCorrelationScenario}'s own header for what each ingredient reproduces. + * + *

    Diagnostics on failure: unlike the pod-debugging loop this scenario exists to + * replace, the child's {@code TEST_LOG} stream (queue depths, batch sizes, interception + * counts - the same evidence collected off a live pod's logs) arrives on this process's + * stdout consumer. A failing run embeds a filtered tail of it in the assertion message, so + * the failure mode is diagnosable without redeploying anything. + */ +@Tag("slow") +public class LeakTagCorrelationReferenceChainTest extends AbstractProcessProfilerTest { + + // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE) - the contract under + // test, asserted against the runtime event values the child reports. + private static final long LEAK_TAG_BASE = 0x40000000L; + private static final long LEAK_TAG_POOL_SIZE = 256; + + @Test + void shouldCorrelateLeakChainTargetTagWithLiveObjectLeakTag() throws Exception { + assertCorrelationFound("pausetarget=60000"); + } + + /** + * The pod-regime variant of the correlation use case: the same scenario under a + * pod-like tight pass window (pausetarget in single-digit ms, as on the production-like + * pod whose expand window was ~10ms against a 14-41ms GetObjectsWithTags floor at a + * 242k-entry tag map, ev-leaktag-onpod-round4). Full correlation cannot be the pass + * criterion at this window width: the scenario's trend gates were tuned for the loose + * window's crawl/GC interleaving, and under tight windows the leak klass's real-fold + * trend can flip within a single run - the same knife-edge the pod itself lives on + * (chains arrive via the plain crawl path while pool tagging/interception loses the + * race; observed: [correlation-tag-out-of-pool] with zero interceptions). What CAN be + * asserted deterministically here is the machinery this window exists to exercise: + * the leak-accumulation rotation must engage and select FRONTIER-state (un-expanded + * backlog) holders - the round-4 fix whose absence made that tier select zero on the + * pod while holding 51k known parent candidates. GetObjectsWithTags batch control is + * covered unit-level (gotwWindowNs() widening): a real child JVM's local tag map + * (~100k filler) keeps the per-call floor under the nominal window, so the widening + * regime cannot be reproduced end-to-end locally without a 242k-entry tag map. + */ + @Test + void shouldRotateFrontierLeakHoldersUnderPodLikeTightWindows() throws Exception { + assertCorrelationFound("pausetarget=10", /*requireFrontierRotation=*/ true); + } + + /** + * Drives the leak-correlation child JVM and asserts the correlated chain, with the + * referencechains pausetarget given by {@code pauseTargetOption} (everything else held + * fixed across the two callers so the variant differs in exactly the pass-window regime). + */ + private void assertCorrelationFound(String pauseTargetOption) throws Exception { + assertCorrelationFound(pauseTargetOption, /*requireFrontierRotation=*/ false); + } + + /** + * Drives the leak-correlation child JVM and asserts the correlated chain, with the + * referencechains pausetarget given by {@code pauseTargetOption} (everything else held + * fixed across the two callers so the variant differs in exactly the pass-window regime). + * With {@code requireFrontierRotation} (the tight-window pod-regime variant), the + * leak-accumulation rotation's FRONTIER-state selection is additionally asserted, and + * the correlation result markers that the pod regime legitimately produces without the + * full correlation completing (not-found, tag-out-of-pool - see the tight variant's own + * comment) are accepted instead of failed. + */ + private void assertCorrelationFound(String pauseTargetOption, + boolean requireFrontierRotation) throws Exception { + assumeFalse(Platform.isJavaVersion(8)); + assumeFalse(Platform.isJ9()); + assumeFalse(Platform.isZing()); + + Path scratchDumpPath = Files.createTempFile("referencechains-correlation-", ".jfr"); + Files.deleteIfExists(scratchDumpPath); + Path continuousJfrPath = Files.createTempFile("referencechains-correlation-continuous-", ".jfr"); + // Rolling tail of the child's TEST_LOG stream for failure diagnostics - bounded so a + // long run cannot grow this without limit. + Deque testLogTail = new ArrayDeque<>(); + AtomicBoolean frontierRotationSeen = new AtomicBoolean(false); + try { + // Same raised budget as ExternalProcessReferenceChainTest's own scenarios: a genuinely + // fresh, cold external JVM's reachable graph is far larger than the in-process tests', + // and this scenario adds ~100k filler objects on top. The pausetarget varies by caller. + // memory=64:l:1.0 - the explicit liveness-sample ratio is load-bearing, + // not decoration: without a ratio segment, the default is 10% + // (arguments.h's _live_samples_ratio) and LivenessTracker::track() + // drops 90% of tracked instances probabilistically. This scenario's + // leak cohort is only tens of 1MB chunks, so the default makes the + // (klass, tid) population a per-run Bernoulli lottery - observed live + // as the intermittent "tagLeakInstances ZERO" runs where zero chunks + // survived the filter (passing runs' stable "tagged=5" is exactly the + // ~10% tail of ~50 chunks). 1.0 makes the scenario's documented + // assumption ("every allocation is sampled, tracked, and taggable", + // LeakTagCorrelationScenario) actually true. + String startCommand = "start,memory=64:l:1.0,generations=true," + + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:" + + pauseTargetOption + + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); + String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); + + List jvmArgs = Collections.singletonList( + "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + + AtomicReference resultLine = new AtomicReference<>(); + LaunchResult result = launch("leak-correlation", jvmArgs, packedCommand, + Collections.emptyMap(), + // The scenario's own round loop (up to 25 rounds plus a 5-attempt grace period, + // ~40-60s observed) on top of a cold external JVM's startup/classloading cost and + // this scenario's larger graph - same margin logic as the existing scenarios. + 150, + line -> { + if (line.startsWith(LeakTagCorrelationScenario.FOUND_MARKER) + || line.equals(LeakTagCorrelationScenario.NOT_FOUND_MARKER) + || line.startsWith(LeakTagCorrelationScenario.TAG_OUT_OF_POOL_MARKER) + || line.startsWith(LeakTagCorrelationScenario.NO_LIVE_OBJECT_MARKER) + || line.startsWith(LeakTagCorrelationScenario.TRANSIENT_CHAIN_MARKER) + || line.startsWith(LeakTagCorrelationScenario.NOISE_CHAIN_MARKER)) { + resultLine.set(line); + } + if (line.startsWith("[TEST::INFO]")) { + if (line.contains("collectLeakAccumulationCandidatesForRotation selected") + && line.contains("state=FRONTIER")) { + frontierRotationSeen.set(true); + } + testLogTail.addLast(line); + while (testLogTail.size() > 2000) { + testLogTail.removeFirst(); + } + } + return LineConsumerResult.CONTINUE; + }, + null); + + assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); + assertNotNull(resultLine.get(), "Child process never printed a recognizable result marker" + + " on stdout" + diagnostics(testLogTail)); + String line = resultLine.get(); + if (requireFrontierRotation) { + assertTrue(frontierRotationSeen.get(), + "leak-accumulation rotation never selected a FRONTIER-state (un-expanded) " + + "holder under the pod-like tight window - the round-4 regression this " + + "variant exists to catch (tier dead: zero selections) would look exactly " + + "like this" + diagnostics(testLogTail)); + } + if (line.startsWith(LeakTagCorrelationScenario.FOUND_MARKER)) { + long targetTag = Long.parseLong( + line.substring(LeakTagCorrelationScenario.FOUND_MARKER.length()).trim()); + assertTrue(targetTag >= LEAK_TAG_BASE && targetTag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE, + "Reported targetTag " + targetTag + " is outside the leak-tag pool range [" + + LEAK_TAG_BASE + ", " + (LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE) + ")"); + return; + } + if (requireFrontierRotation && (line.equals(LeakTagCorrelationScenario.NOT_FOUND_MARKER) + || line.startsWith(LeakTagCorrelationScenario.TAG_OUT_OF_POOL_MARKER))) { + // The pod regime legitimately produces these without full correlation completing + // (see the tight variant's comment); the loose variant still fails on them. + return; + } + fail("Leak-tag correlation did not succeed: " + line + diagnostics(testLogTail)); + } finally { + Files.deleteIfExists(scratchDumpPath); + Files.deleteIfExists(continuousJfrPath); + } + } + + /** + * A filtered summary of the child's TEST_LOG stream for failure messages: the last few + * pass summaries (queue depths are the first thing to check on any crawl stall), the + * gotw batch-size trace (batch control regressions), and any leak-tag interception or + * correlation lines. Capped so the message stays readable. + */ + private static String diagnostics(Deque testLogTail) { + if (testLogTail.isEmpty()) { + return "\n(no TEST_LOG output captured - non-debug build or no reference-chain passes ran)"; + } + List runPassDone = new ArrayList<>(); + List gotw = new ArrayList<>(); + List leakTagLines = new ArrayList<>(); + for (String s : testLogTail) { + if (s.contains("runPass done:")) { + runPassDone.add(s); + } else if (s.contains("expandFrontier gotw")) { + gotw.add(s); + } else if (s.contains("intercepted") || s.contains("correlateAdmittedLeakTag") + || s.contains("recordDiscoveredInstance") || s.contains("auto-mark") + || s.contains("rotation_candidates") + || s.contains("collectLeakAccumulationCandidatesForRotation selected")) { + leakTagLines.add(s); + } + } + StringBuilder sb = new StringBuilder("\n--- child diagnostics (filtered TEST_LOG) ---"); + appendLast(sb, "runPass done", runPassDone, 5); + appendLast(sb, "gotw", gotw, 5); + appendLast(sb, "leak-tag activity", leakTagLines, 20); + return sb.toString(); + } + + private static void appendLast(StringBuilder sb, String label, List lines, int max) { + sb.append("\n").append(label).append(" (last ").append(max).append(" of ") + .append(lines.size()).append("):"); + int from = Math.max(0, lines.size() - max); + for (int i = from; i < lines.size(); i++) { + sb.append("\n ").append(lines.get(i)); + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java new file mode 100644 index 0000000000..f83eb16c15 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -0,0 +1,510 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.JavaProfiler; +import org.openjdk.jmc.common.IMCType; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.IType; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.ArrayList; +import java.util.List; + +/** + * The "leak-tag correlation" use case, run inside a genuinely separate child JVM by + * {@code ExternalLauncher}'s {@code leak-correlation} mode: a growing static-field + * collection leak, coexisting with (a) concurrent, ephemeral, stack-local allocations of + * another class and (b) a large live object graph, must end up reported as a + * {@code datadog.ReferenceChain} event whose {@code targetTag} is a leak-tag-pool tag, + * matching a {@code datadog.HeapLiveObject} event's {@code leakTag} for the same object - + * the chain-to-live-heap-sample correlation the whole leak-tag pool exists to provide. + * + *

    Each ingredient exists to reproduce, in miniature, a condition that only showed + * against real large deployments before this scenario existed (all observed live on a + * production-like pod, each escaping every earlier local test): + *

      + *
    • Scale ({@link #FILLER}): ~100k live filler objects grow the JVMTI tag map + * enough that {@code GetObjectsWithTags} has a real per-call cost floor, the BFS + * frontier/backlog has real depth, and per-pass deadlines actually bind - the regime + * where batch-size control, fair-share queue drain, and rotation starvation bugs live. + * Built (like the leak itself) BEFORE the profiler starts, per + * {@code LeakingCacheScenario}'s own comment: the one-shot root-seeded walk fires ~1s + * after start and must never catch the fixture half-built.
      • + *
    • Noise ({@link NoisePayload} on the noise thread): short-lived allocations + * held only by a live thread's stack frame during that same one-shot walk - admitted + * as {@code stack_local}-rooted depth-1 frontier entries, exactly the transient-root + * noise shape the discovered-chain gate exists to suppress. The scenario asserts no + * such chain is ever emitted; a regression in the gate emits one and fails this + * scenario with {@link #TRANSIENT_CHAIN_MARKER}.
      • + *
    • Correlation (the actual acceptance signal): a leaked {@code byte[]} + * chunk's chain must carry a {@code targetTag} in the leak-tag pool's range - + * mirrored here from {@code LEAK_TAG_BASE}/{@code LEAK_TAG_POOL_SIZE} + * (referenceChains.h) - and at least one {@code datadog.HeapLiveObject} event for a + * {@code byte[]} must carry the same value in {@code leakTag}.
      • + *
    + * + *

    Same separate-process rationale as {@code LeakingCacheScenario}'s own class comment + * (the one-shot root-seeded walk is a per-process resource), and same seed-before-start + * and debug-seam patterns as {@code StaticFieldGrowingCollectionScenario}. + */ +public final class LeakTagCorrelationScenario { + private LeakTagCorrelationScenario() {} + + /** + * The leak: a static collection appended to for the process's whole life - the + * canonical unmaintained-singleton retention shape, DELIBERATELY a growing + * {@code ArrayList} and not a fixed holder. An ArrayList's backing + * {@code elementData} array is REPLACED on growth, and an EXPANDED frontier node's + * children set is frozen at expansion time (expandFrontier()'s own comment), so + * discovering contents added after a resize is exactly the hard production case + * this scenario must prove works: the live holder's re-walk has to admit each new + * backing array, and only then can its new chunks be admitted and leak-tag + * intercepted. (An earlier fixed-array version of this fixture sidestepped that + * requirement and still failed - see the scenario history in the investigation + * notes - which is what exposed the rotation starvation this fixture now encodes.) + */ + static final List LEAK_SINK = new ArrayList<>(); + + /** + * Scale ingredient: a static tree of plain objects, unreachable from anywhere but this + * field - exists only to give the frontier/backlog and the JVMTI tag map real depth. + */ + static final Object[][] FILLER = new Object[1500][]; + + // Arbitrary, scenario-chosen klass ids - see LeakingCacheScenario.CACHED_PAYLOAD_TEST_KLASS_ID's + // own comment; distinct from the other scenarios' ids (987301/987302) since all may run in the + // same suite, each in its own separate child JVM. + private static final int LEAK_KLASS_ID = 987303; + private static final int NOISE_KLASS_ID = 987304; + + // See LeakingCacheScenario.SEED_EPOCHS_FOR_HYSTERESIS's own comment for the full derivation. + private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; + + private static final int SEED_CHUNKS = 10; + + /** + * Per-round trend-maintenance epoch counter (debug builds only) - same + * pattern and rationale as StaticFieldGrowingCollectionScenario's own: + * a one-shot seeded ramp ages out of LivenessTracker's hysteresis once + * enough real fold samples interleave (observed live: the byte[] candidate + * dropped right after the first leak-tag interceptions, stranding the + * correlated discoveries with no poll loop to build their chains), so + * each round pushes one fresh rising epoch to keep the candidates alive. + */ + private static int hysteresisEpoch = 0; + + private static void addChunk() { + LEAK_SINK.add(new byte[CHUNK_BYTES]); + } + + // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE): the pool range a + // correlated chain's targetTag must fall into, and the range a matching HeapLiveObject's + // leakTag must fall into. This is the contract under test, asserted against runtime + // event values - not constants compared to constants. + private static final long LEAK_TAG_BASE = 0x40000000L; + private static final long LEAK_TAG_POOL_SIZE = 256; + + /** + * Transient handoff channel for the noise representative - deliberately a local of + * {@code run()} (captured by the noise thread's lambda), NOT a static field: any static + * field holding a {@link NoisePayload} would make that one instance genuinely, + * durably static-retained, and the profiler would be RIGHT to emit a static-rooted chain + * for it - found the hard way on this scenario's very first run, which correctly reported + * {@code NoisePayload@depth0:static_field}. After the handoff the only JVM-heap reference + * to the representative is the noise thread's live frame, which is exactly the + * transient-rooted retention this class exists to exercise. + */ + private static final class NoiseHandoff { + volatile NoisePayload representative; + // The noise thread's own profiler tid - published alongside the payload so + // the seeding below can qualify exactly that thread for the noise klass + // (selectLeakCandidates()'s per-(klass, tid) gate matches the tid that + // actually allocated the noise instances, and that is this thread). + volatile int tid; + + void publish(NoisePayload payload, int tid) { + // tid FIRST (volatile ordering): a caller that sees `representative` set + // is then also guaranteed to see the tid published with it. + this.tid = tid; + representative = payload; + } + + NoisePayload take() { + NoisePayload payload = representative; + representative = null; // drop this path's only reference - retention stays transient + return payload; + } + } + + /** Printed to stdout, followed by the correlated targetTag, on full success. */ + public static final String FOUND_MARKER = "[correlation-found] "; + + /** Printed to stdout if no leaked byte[] chain was ever observed. */ + public static final String NOT_FOUND_MARKER = "[correlation-not-found]"; + + /** + * Printed to stdout (with the offending tag) if a leaked byte[] chain was emitted but its + * targetTag never landed in the leak-tag pool's range - the chain-to-live-heap + * correlation's first half failing. + */ + public static final String TAG_OUT_OF_POOL_MARKER = "[correlation-tag-out-of-pool] "; + + /** + * Printed to stdout (with the targetTag) if a correlated byte[] chain was emitted but + * no datadog.HeapLiveObject event for a byte[] ever carried the same leakTag - the + * correlation's second half failing (tagging and live-heap recording disagreeing on + * which instances are the tracked leak candidates). + */ + public static final String NO_LIVE_OBJECT_MARKER = "[correlation-no-live-object] "; + + /** + * Printed to stdout (with a summary of every violation) if any ReferenceChain event was + * emitted whose root kind is a transient root (stack/jni local) at depth <= 1 - the + * discovered-chain gate's exact suppression contract (depth==0 always, depth==1 when + * transient-rooted); anything matching it reaching the output is a gate regression. + */ + public static final String TRANSIENT_CHAIN_MARKER = "[correlation-transient-chain-emitted] "; + + /** + * Printed to stdout (with a summary of the offending chains) if the deliberately ephemeral + * {@link NoisePayload} - held only by the noise thread's stack frame - ever got a chain + * emitted, whether via the transient gate or any other path: nothing about it is durably + * retained, so it must never be reported as a leak. + */ + public static final String NOISE_CHAIN_MARKER = "[correlation-noise-chain-emitted] "; + + /** One dump's scan outcome - all fields per the latest dump only. */ + private static final class ScanResult { + ReferenceChainAssertions.ChainMatch payloadChain; + boolean payloadChainInPool; + int liveObjectMatchesForChainTag; + final List transientViolations = new ArrayList<>(); + final List noiseChains = new ArrayList<>(); + } + + public static void run(JavaProfiler profiler, String startCommand, Path scratchDumpPath) + throws Exception { + // Build the whole fixture before the profiler starts - see the class comment and + // LeakingCacheScenario's own seed-before-start rationale. + for (int i = 0; i < SEED_CHUNKS; i++) { + addChunk(); + } + for (int row = 0; row < FILLER.length; row++) { + Object[] column = new Object[64]; + for (int col = 0; col < column.length; col++) { + column[col] = new FillerNode(); + } + FILLER[row] = column; + } + NoiseHandoff handoff = new NoiseHandoff(); + Thread noiseThread = new Thread(() -> { + // Held ONLY by this live frame across the process's whole life, so the one-shot + // root-seeded walk (a few seconds away) admits them as stack_local-rooted entries. + NoisePayload[] held = new NoisePayload[64]; + for (int i = 0; i < held.length; i++) { + held[i] = new NoisePayload(); + } + // The tid must be captured on THIS thread: the per-(klass, tid) seeding below + // qualifies the allocating thread, and the noise instances are all allocated here. + handoff.publish(held[0], JavaProfiler.getTid()); + long slot = 0; + while (true) { + try { + Thread.sleep(50); + } catch (InterruptedException e) { + return; + } + // Churn one slot at a time: the population stays live-but-ephemeral (constant live + // count, nothing durably retained beyond the frame) while still allocating. + held[(int) (slot++ % held.length)] = new NoisePayload(); + } + }, "leak-correlation-noise"); + noiseThread.setDaemon(true); + noiseThread.start(); + + if (startCommand != null && !startCommand.isEmpty()) { + profiler.execute(startCommand); + } + + boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + // Declared at method scope: the per-round and final-attempt maintenance + // seeding below (in their own debugBuild blocks) reuse these tids. + int leakTid = 0; + int noiseTid = 0; + if (debugBuild) { + // Representatives FIRST, seeds second: since the leak-tag pool redesign, candidate + // matching, leak-tag assignment (tagLeakInstances scanning the live-heap tracking + // table), and discovered-instance recording are all keyed on the REAL klass id - + // setKlassPopulationRepresentativeForTest0 resolves each representative's real id + // and aliases the synthetic id to it, and the hysteresis seeds below then land in + // the real entry (seeding first would leave them under the synthetic id, where they + // authorize nothing but hasLeakSignal's candidate-count check). + // The leak representative is durably reachable via LEAK_SINK for the rest of the + // process's life; the noise representative only via the noise thread's frame (the + // handoff's own reference is dropped by take()) - exactly what makes one a valid + // leak candidate and the other a gate test. + JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, LEAK_SINK.get(0)); + JavaProfiler.setKlassPopulationRepresentativeForTest0(NOISE_KLASS_ID, handoff.take()); + // Per-(klass, tid) qualification seeds: selectLeakCandidates() now also + // requires a qualifying ALLOCATING THREAD, and tagLeakInstances() scopes leak + // tags to exactly those tids' instances - so the seeded qualification must + // name the REAL allocating tids (this thread for the leak chunks, the noise + // thread for the noise payloads) or no tracked instance would ever match + // the tagging scope. Same epoch-aligned ramp shape as the klass seeds + // (scaled to fit the per-tid u8 ring); the synthetic flag keeps these + // ramps immune to the real fold's absent-tid decay between rounds. + leakTid = JavaProfiler.getTid(); + noiseTid = handoff.tid; + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedKlassPopulationSample0(NOISE_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); + JavaProfiler.seedTidTrendSample0(NOISE_KLASS_ID, noiseTid, epoch * 3, epoch); + } + // Wait for the BFS thread to fully complete its first pass (static sweep AND frontier + // expansion) before any growth round - same load-bearing wait and rationale as + // StaticFieldGrowingCollectionScenario: the late chunks added afterward must be + // discovered via rotation re-walks, not by a first pass that has not even run yet. + int initialPasses = JavaProfiler.referenceChainPassesRunForTest0(); + for (int i = 0; i < 300; i++) { + if (JavaProfiler.referenceChainPassesRunForTest0() != initialPasses) { + break; + } + Thread.sleep(100); + } + JavaProfiler.pollReferenceChainTargets0(); + } else { + // No debug seams outside debug builds - give the real allocation sampler and the + // pass loop time to notice the seeded population on their own. + Thread.sleep(5000); + } + + // Each dump re-emits every resolved chain (drainPendingChainEvents()'s + // snapshot-and-keep contract), so a later dump's scan is a superset state - the + // latest scan with a payload chain observed is always the best evidence. + ScanResult best = null; + int totalRounds = 25; + // Slow growth per round is deliberate: each chunk clears the allocation + // sampler's floor on its own (see CHUNK_BYTES), so every allocation is + // sampled, tracked, and taggable - the per-round population growth is + // only the trend signal, not a sampling-volume requirement. + for (int round = 1; round <= totalRounds; round++) { + addChunk(); + if (debugBuild) { + hysteresisEpoch++; + JavaProfiler.seedKlassPopulationSample0( + LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedKlassPopulationSample0( + NOISE_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0( + LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0( + NOISE_KLASS_ID, noiseTid, hysteresisEpoch * 3, hysteresisEpoch); + } + System.gc(); + profiler.dump(scratchDumpPath); + // Scan only every 3rd round: each scan's JMC parse allocates megabytes + // on this same JVM, and that churn races the leak chunks for space in + // LivenessTracker's tracking table (the age-priority tagging then + // selects old JVM-machinery survivors instead of the young chunks - + // observed live as tagged=5 stable but zero interceptions). Less + // scan churn gives the chunks time to accumulate surviving ages and + // win the tagging priority. Rounds are still dumped every time so + // the final dump always holds the freshest chains. + if (round % 3 == 0) { + best = keepBest(best, scan(scratchDumpPath)); + if (best != null && best.payloadChainInPool + && best.liveObjectMatchesForChainTag > 0) { + break; // full correlation achieved - no need to grow further + } + } + Thread.sleep(300); + } + + for (int attempt = 0; attempt < 5; attempt++) { + if (best != null && best.payloadChainInPool && best.liveObjectMatchesForChainTag > 0) { + break; + } + if (debugBuild) { + hysteresisEpoch++; + JavaProfiler.seedKlassPopulationSample0( + LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedKlassPopulationSample0( + NOISE_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0( + LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0( + NOISE_KLASS_ID, noiseTid, hysteresisEpoch * 3, hysteresisEpoch); + } + Thread.sleep(1000); + System.gc(); + profiler.dump(scratchDumpPath); + best = keepBest(best, scan(scratchDumpPath)); + } + + if (best == null || best.payloadChain == null) { + System.out.println(NOT_FOUND_MARKER); + return; + } + if (!best.transientViolations.isEmpty()) { + System.out.println(TRANSIENT_CHAIN_MARKER + best.transientViolations); + return; + } + if (!best.noiseChains.isEmpty()) { + System.out.println(NOISE_CHAIN_MARKER + best.noiseChains); + return; + } + if (!best.payloadChainInPool) { + System.out.println(TAG_OUT_OF_POOL_MARKER + best.payloadChain.targetTag); + return; + } + if (best.liveObjectMatchesForChainTag == 0) { + System.out.println(NO_LIVE_OBJECT_MARKER + best.payloadChain.targetTag); + return; + } + System.out.println(FOUND_MARKER + best.payloadChain.targetTag); + } + + /** Latest scan wins; a scan without a payload chain never displaces one that had it. */ + private static ScanResult keepBest(ScanResult best, ScanResult scan) { + if (scan == null) { + return best; + } + if (scan.payloadChain != null) { + return scan; + } + return best != null ? best : scan; + } + + /** + * Scans one dump: the first leaked {@code byte[]} chain (if any), whether its targetTag is + * in the leak-tag pool, how many {@code datadog.HeapLiveObject} events for a + * leaked {@code byte[]} carry that same tag in {@code leakTag}, plus the two gate + * regressions (transient-rooted depth<=1 chains, any {@link NoisePayload} chain). + */ + private static ScanResult scan(Path scratchDumpPath) throws Exception { + if (!Files.exists(scratchDumpPath)) { + return null; + } + IItemCollection events; + try (InputStream in = Files.newInputStream(scratchDumpPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + ScanResult result = new ScanResult(); + + IItemCollection chains = events.apply( + org.openjdk.jmc.common.item.ItemFilters.type("datadog.ReferenceChain")); + for (IItemIterable iterable : chains) { + IType type = iterable.getType(); + IMemberAccessor chainAccessor = ReferenceChainAssertions.findAccessor(type, "chain"); + IMemberAccessor targetTagAccessor = + ReferenceChainAssertions.findAccessor(type, "targetTag"); + IMemberAccessor depthAccessor = ReferenceChainAssertions.findAccessor(type, "depth"); + IMemberAccessor rootKindAccessor = + ReferenceChainAssertions.findAccessor(type, "rootKind"); + for (IItem item : iterable) { + Object chainValue = chainAccessor.getMember(item); + if (!(chainValue instanceof Object[])) { + continue; + } + Object[] rawChain = (Object[]) chainValue; + String leaf = rawChain.length > 0 && rawChain[0] instanceof IMCType + ? ((IMCType) rawChain[0]).getFullName() : ""; + long targetTag = targetTagAccessor != null + ? ReferenceChainAssertions.numberValue(targetTagAccessor.getMember(item)) : -1; + int depth = depthAccessor != null + ? (int) ReferenceChainAssertions.numberValue(depthAccessor.getMember(item)) : -1; + Object rootKind = rootKindAccessor != null ? rootKindAccessor.getMember(item) : null; + String rootKindName = rootKind != null ? rootKind.toString() : ""; + if (rootKindName.startsWith("first_observed_via:") && depth <= 1) { + result.transientViolations.add(leaf + "@depth" + depth + ":" + rootKindName); + } + if (NoisePayload.class.getName().equals(leaf)) { + result.noiseChains.add(leaf + "@depth" + depth + ":" + rootKindName); + } + if ("byte[]".equals(leaf)) { + boolean inPool = + targetTag >= LEAK_TAG_BASE && targetTag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE; + // Multiple byte[] chains exist in a single dump: every discovered + // instance of the candidate class gets one, and only the tracked-and- + // tagged instances carry a pool-range targetTag - the rest legitimately + // carry their frontier tag (the correlation is additive on top of the + // chain, not a property of every chain). Prefer the first in-pool + // (correlated) chain over any out-of-pool one seen earlier. + if ((inPool && !result.payloadChainInPool) || result.payloadChain == null) { + List chain = new ArrayList<>(rawChain.length); + for (Object element : rawChain) { + chain.add((IMCType) element); + } + result.payloadChain = new ReferenceChainAssertions.ChainMatch( + chain, targetTag, depth); + result.payloadChainInPool = inPool; + } + } + } + } + + if (result.payloadChainInPool) { + IItemCollection liveObjects = events.apply( + org.openjdk.jmc.common.item.ItemFilters.type("datadog.HeapLiveObject")); + for (IItemIterable iterable : liveObjects) { + IType type = iterable.getType(); + IMemberAccessor leakTagAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTag"); + IMemberAccessor objectClassAccessor = + ReferenceChainAssertions.findAccessor(type, "objectClass"); + for (IItem item : iterable) { + long leakTag = leakTagAccessor != null + ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; + if (leakTag != result.payloadChain.targetTag) { + continue; + } + Object objectClass = objectClassAccessor != null + ? objectClassAccessor.getMember(item) : null; + if (objectClass instanceof IMCType + && "byte[]".equals(((IMCType) objectClass).getFullName())) { + result.liveObjectMatchesForChainTag++; + } + } + } + } + return result; + } + + /** + * Chunk size that clears ObjectSampler's real 256KiB sampling floor on its own - same + * rationale as StaticFieldGrowingCollectionScenario.CHUNK_BYTES. Load-bearing here: + * every chunk allocation must be allocation-sampled so it becomes a live-heap tracked + * instance (the only instances LivenessTracker's leak-tag pool tags, and the only ones + * that get datadog.HeapLiveObject events to correlate against). A small leaked object + * only gets sampled probabilistically, which made the correlation dependent on + * sampling luck - observed live as pass-standalone/fail-in-suite before this. + */ + private static final int CHUNK_BYTES = 1_000_000; + + /** + * The noise object: deliberately ephemeral - held only by the noise thread's live + * frame, never by any field, so nothing about it is durably retained. + */ + static final class NoisePayload { + long p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15; + long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31; + } + + /** Scale-only object: no references, just weight - one frontier entry each. */ + static final class FillerNode { + long p0, p1, p2, p3, p4, p5, p6, p7; + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java new file mode 100644 index 0000000000..ed47fe10f3 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java @@ -0,0 +1,353 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.JavaProfiler; +import org.openjdk.jmc.common.IMCType; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +/** + * The actual "leaking Java app" body run inside a genuinely separate child JVM by + * {@code ExternalLauncher}'s {@code leak-cache} mode, driven by + * {@code ExternalProcessReferenceChainTest}. Deliberately not a JUnit test itself - no + * {@code AbstractProfilerTest}/dynamic-attach lifecycle, no shared-JVM {@code ReferenceChainTracker} + * singleton with any other test - this class's whole point is to run alone in its own process, so + * the one-shot root-seeded BFS walk (see this class's own comment on why a previous attempt at + * folding this scenario into {@code ReferenceChainTrackingTest} as a second {@code @Test} method + * failed) always belongs to this scenario and only this scenario. + * + *

    Same realistic-leak shape as {@code ReferenceChainTrackingTest}'s own (now-removed) in-process + * attempt: an ever-growing, never-evicted {@code HashMap}-backed cache, held by a local variable. + * + *

    Why {@code cache} is seeded and populated before the profiler is even started + * (found the hard way, against a real separate-process run): {@code threadLoop()} + * (referenceChains.cpp) calls {@code OS::sleep(_effective_cadence_ns)} - 1 second, initially - + * *before* its very first {@code shouldRunPass()} check, and {@code shouldRunPass()} always + * returns {@code true} on that first-ever check ({@code !_search_started}). So the process's + * one-and-only root-seeded walk fires roughly one second after {@code startThread()}, or earlier + * if woken by a GC-finish signal - a race {@code ReferenceChainTrackingTest}'s own + * {@code shouldReportAbandonedSearchOnTinyFrontierCap()} comment already documents ("that wakeup + * can race the thread's own startup ... and be missed"). An initial in-process attempt at this + * exact scenario created {@code cache} and started allocating only *after* starting the profiler + * (mirroring {@code ReferenceChainTrackingTest}'s {@code gcRootHolder}, which happened to work by + * luck of scheduling in a warm, already-running JVM) - against a genuinely fresh, cold JVM this + * lost the race essentially every time: the one-shot walk caught {@code cache} still empty, + * permanently blocking discovery of everything added to it afterward (a frontier entry once + * marked {@code EXPANDED} is never revisited for new children - see {@code expandFrontier()}'s + * own comment). Seeding {@code cache} with real data *before* {@code profiler.execute()} is even + * called removes the race entirely: no matter how fast the walk fires, {@code cache} is never + * empty when it does. + */ +public final class LeakingCacheScenario { + private LeakingCacheScenario() {} + + // Arbitrary, scenario-chosen klass id - this process's own LivenessTracker population table + // treats it as an opaque key (see KlassPopulationEntry's own comment) and this process never + // runs any other reference-chains scenario, so no collision risk with e.g. + // ReferenceChainTrackingTest's/ReferenceChainTestSeamsTest's own klass ids in their own, + // separate JVMs. + private static final int CACHED_PAYLOAD_TEST_KLASS_ID = 987301; + + // See ReferenceChainTrackingTest.SEED_EPOCHS_FOR_HYSTERESIS's own comment for the full + // derivation: LivenessTracker's hysteresis gate (livenessTracker.h's + // KLASS_POPULATION_MIN_FILL_FOR_TREND=10 / LEAK_TREND_HYSTERESIS_BASE=5) only starts + // incrementing consecutive_positive once ring_fill has reached 10, so a batch of exactly 10 + // seedKlassPopulationSample0() calls caps consecutive_positive at 1 - far short of what + // selectLeakCandidates() (livenessTracker.cpp) requires before ReferenceChainTracker:: + // hasLeakSignal() reports a candidate, which is what this scenario's own generations=true + // startCommand needs to authorize the BFS thread's first real pass at all. + private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; + + /** + * Durable holder for the cache fixture. The discovered-chain gate suppresses + * chains shallower than the first real holder hop that are rooted at a TRANSIENT + * root (stack/JNI local) - a frame-held "leak" is by definition not a leak, so the + * cache must be retained through a static field (the real singleton-collection + * shape) rather than only {@code run()}'s own local, which is exactly the + * transient-rooted shape the gate suppresses. {@code run()}'s local {@code cache} + * variable below aliases this static; the durable root is what makes the emitted + * chain durable-rooted. + */ + private static final Map CACHE = new HashMap<>(); + + /** Printed to stdout, followed by the matched leaf class's name, on success. */ + public static final String FOUND_MARKER = "[chain-found] "; + + /** Printed to stdout (with no class name suffix) if no match was ever observed. */ + public static final String NOT_FOUND_MARKER = "[chain-not-found]"; + + /** + * Printed to stdout (with the offending class name appended) if a match was found but its + * chain did not pass through {@code java.util.HashMap}'s own internal storage - see + * {@code ExternalProcessReferenceChainTest} for why this is asserted in the parent process + * rather than here (keeping this scenario's own stdout contract to "found/not-found" and + * leaving richer chain-shape assertions to the JUnit side, the same separation of concerns + * {@code ReferenceChainJfrParserTest} already uses for the lower-level JFR-format proof). + */ + public static final String NO_HASHMAP_INTERNALS_MARKER = "[chain-found-no-hashmap-internals] "; + + /** + * Printed to stdout (with an "N/M" re-emitted-dump count appended) if the chain was + * successfully reconstructed but did not re-emit into one or more subsequent, + * independently written JFR dumps - i.e. the "snapshot-and-keep" contract of + * {@code ReferenceChainTracker::drainPendingChainEvents()} (referenceChains.cpp), which + * {@code Profiler::dump()} invokes on every dump without clearing the resolved-chain cache, + * failed to hold across dumps. Asserted by {@code ExternalProcessReferenceChainTest}. + */ + public static final String CHAIN_NOT_PERSISTED_MARKER = "[chain-not-persisted] "; + + /** + * Seeds {@code cache} with an initial batch, *then* starts the profiler (see this class's own + * comment for why that order is load-bearing), then runs the same population-growth loop + * {@code ReferenceChainTrackingTest} pioneered - dumping to {@code scratchDumpPath} after every + * round (and a short grace period beyond the last round) until a {@code datadog.ReferenceChain} + * event whose {@code chain[0]} is {@link CachedPayload} appears - then prints exactly one + * result line to stdout. Called from {@code ExternalLauncher} (a different package - hence + * {@code public}), which owns the generic child-process bootstrap (JVMTI/library init, process + * exit) this scenario deliberately stays agnostic of; starting the profiler itself happens + * here, not in {@code ExternalLauncher}, specifically so {@code cache} can be seeded first. + */ + public static void run(JavaProfiler profiler, String startCommand, Path scratchDumpPath) throws Exception { + Map cache = CACHE; + seedInitialBatch(cache, 300); + System.out.println("[debug] startCommand=" + startCommand); + System.out.println("[debug] scratchDumpPath=" + scratchDumpPath); + if (startCommand != null && !startCommand.isEmpty()) { + profiler.execute(startCommand); + } + System.out.println("[debug] profiler.execute() returned"); + + boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + boolean seededTestKlassTrend = false; + ReferenceChainAssertions.ChainMatch match = null; + int totalRounds = 25; + for (int round = 1; round <= totalRounds && match == null; round++) { + int newEntries = round * 300; + int roundNumber = round; + Thread allocator = new Thread(() -> { + for (int i = 0; i < newEntries; i++) { + String key = "leak-" + roundNumber + "-" + i; + cache.put(key, new CachedPayload(key)); + } + }); + allocator.start(); + allocator.join(); + System.gc(); + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + + if (match == null && debugBuild) { + // Same debug-only, seeded-representative short-circuit as ReferenceChainTrackingTest's + // own shouldReconstructReferrerChainThroughUnboundedCacheLeak() (see that method's own + // comment, and SEED_EPOCHS_FOR_HYSTERESIS's own comment above): decouples this scenario's + // assertion from whether the real, probabilistic allocation-sampling-driven slope + // detection happens to notice CachedPayload's own population trend within this scenario's + // fixed round/timeout budget - by clearing hasLeakSignal()'s own hysteresis gate outright, + // which is what actually authorizes the BFS thread's first real pass, rather than assuming + // some earlier pass has already tagged anything. "seed-0" (from seedInitialBatch()) is + // reachable via cache for the scenario's entire lifetime, so it is a safe, always-valid + // representative regardless of which round this fires on. + // Representative BEFORE seeding: setKlassPopulationRepresentativeForTest0 + // resolves the representative's real klass id and aliases the synthetic id + // to it, so the hysteresis seeds below land in the real population entry + // (candidate matching is real-id keyed since the leak-tag pool redesign). + JavaProfiler.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get("seed-0")); + if (!seededTestKlassTrend) { + // Per-(klass, tid) qualification seeds: the qualifying tid is THIS + // thread's - the creator of "seed-0", the always-reachable representative + // the seeded ramp and the match below hang off. (The per-round allocator + // threads are one-shot, so no single one of their tids carries a sustained + // per-tid signal - exactly the one-cohort-per-thread shape the retained- + // count bar / synthetic seeds exist to keep qualifying.) + int leakTid = JavaProfiler.getTid(); + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedTidTrendSample0(CACHED_PAYLOAD_TEST_KLASS_ID, leakTid, epoch * 3, epoch); + } + seededTestKlassTrend = true; + } + JavaProfiler.pollReferenceChainTargets0(); + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + + if (match == null) { + Thread.sleep(300); + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + if (round == 1 || round == 5 || round == 10 || round == 25) { + System.out.println("[debug] round=" + round + " cache.size()=" + cache.size()); + debugDumpAllChainLeaves(scratchDumpPath); + } + } + + for (int attempt = 0; match == null && attempt < 5; attempt++) { + Thread.sleep(1000); + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + + if (match == null) { + debugDumpAllChainLeaves(scratchDumpPath); + debugDumpCounters(profiler); + System.out.println(NOT_FOUND_MARKER); + return; + } + boolean sawHashMapInternals = false; + for (IMCType type : match.chain) { + if (type.getFullName().startsWith("java.util.HashMap")) { + sawHashMapInternals = true; + break; + } + } + if (!sawHashMapInternals) { + System.out.println(NO_HASHMAP_INTERNALS_MARKER + match.chain); + return; + } + + // Across-dumps persistence: a resolved chain must re-emit into EVERY subsequent dump the + // sample survives into, not only the one dump that first observed it. Profiler::dump() + // re-snapshots the whole resolved-chain cache without clearing it on every call + // (drainPendingChainEvents(), referenceChains.cpp - "snapshot-and-keep, not a drain"), so a + // still-live sample's chain re-emits into every chunk. `cache` still holds every + // CachedPayload here, so the sample stays live across these extra dumps; each dump targets a + // brand-new file, so a chain found there can only be present because it was re-emitted into + // that dump, not left over from the round-loop's own scratchDumpPath. + int persistChecks = 3; + int persisted = 0; + for (int i = 1; i <= persistChecks; i++) { + Path reDumpPath = Files.createTempFile("referencechains-persist-" + i + "-", ".jfr"); + Files.deleteIfExists(reDumpPath); + try { + profiler.dump(reDumpPath); + if (findMatch(reDumpPath) != null) { + persisted++; + } else { + System.out.println("[debug] across-dumps: CachedPayload chain missing from fresh dump #" + i); + } + } finally { + Files.deleteIfExists(reDumpPath); + } + } + if (persisted != persistChecks) { + System.out.println(CHAIN_NOT_PERSISTED_MARKER + persisted + "/" + persistChecks); + return; + } + System.out.println("[debug] across-dumps: CachedPayload chain re-emitted in all " + + persistChecks + " fresh dumps"); + System.out.println(FOUND_MARKER + match.chain.get(0).getFullName()); + } + + /** + * Populates {@code cache} with {@code count} entries under a key namespace ("seed-") disjoint + * from the round loop's own ("leak-") - called before the profiler (and therefore + * {@code ReferenceChainTracker}'s BFS thread) even starts, so the process's one-shot + * root-seeded walk can never catch {@code cache} empty (see this class's own comment on why + * that race is otherwise real). + */ + private static void seedInitialBatch(Map cache, int count) { + for (int i = 0; i < count; i++) { + String key = "seed-" + i; + cache.put(key, new CachedPayload(key)); + } + } + + /** Temporary diagnostic: print all native debug counters. */ + private static void debugDumpCounters(JavaProfiler profiler) { + try { + Map counters = profiler.getDebugCounters(); + System.out.println("[debug] counters (" + counters.size() + " total):"); + counters.entrySet().stream() + .filter(e -> e.getValue() != 0) + .sorted(Map.Entry.comparingByKey()) + .forEach(e -> System.out.println("[debug] " + e.getKey() + " = " + e.getValue())); + } catch (Exception e) { + System.out.println("[debug] exception while dumping counters: " + e); + } + } + + /** Temporary diagnostic: print every datadog.ReferenceChain event's chain[0] class, if any. */ + private static void debugDumpAllChainLeaves(Path scratchDumpPath) { + try { + if (!Files.exists(scratchDumpPath)) { + System.out.println("[debug] scratch dump path does not exist: " + scratchDumpPath); + return; + } + IItemCollection events; + try (InputStream in = Files.newInputStream(scratchDumpPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + IItemCollection chains = events.apply(ItemFilters.type("datadog.ReferenceChain")); + long total = chains.stream().mapToLong(it -> it.getItemCount()).sum(); + System.out.println("[debug] datadog.ReferenceChain total events: " + total); + IItemCollection liveObjects = events.apply(ItemFilters.type("datadog.HeapLiveObject")); + long totalLive = liveObjects.stream().mapToLong(it -> it.getItemCount()).sum(); + System.out.println("[debug] datadog.HeapLiveObject total events: " + totalLive); + IItemCollection abandoned = events.apply(ItemFilters.type("datadog.ReferenceChainAbandoned")); + long totalAbandoned = abandoned.stream().mapToLong(it -> it.getItemCount()).sum(); + System.out.println("[debug] datadog.ReferenceChainAbandoned total events: " + totalAbandoned); + for (org.openjdk.jmc.common.item.IItemIterable iterable : chains) { + org.openjdk.jmc.common.item.IType type = iterable.getType(); + org.openjdk.jmc.common.item.IMemberAccessor chainAccessor = + ReferenceChainAssertions.findAccessor(type, "chain"); + if (chainAccessor == null) { + System.out.println("[debug] no chain accessor found"); + continue; + } + for (org.openjdk.jmc.common.item.IItem item : iterable) { + Object chainValue = chainAccessor.getMember(item); + if (chainValue instanceof Object[]) { + Object[] raw = (Object[]) chainValue; + String leaf = raw.length > 0 && raw[0] instanceof IMCType + ? ((IMCType) raw[0]).getFullName() : ""; + System.out.println("[debug] chain leaf: " + leaf); + } + } + } + } catch (Exception e) { + System.out.println("[debug] exception while dumping chain leaves: " + e); + } + } + + private static ReferenceChainAssertions.ChainMatch findMatch(Path scratchDumpPath) throws Exception { + if (!Files.exists(scratchDumpPath)) { + return null; + } + IItemCollection events; + try (InputStream in = Files.newInputStream(scratchDumpPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + return ReferenceChainAssertions.findMatchForClass( + events.apply(ItemFilters.type("datadog.ReferenceChain")), CachedPayload.class); + } + + /** + * Realistic-leak fixture - identical shape to {@code ReferenceChainTrackingTest}'s own + * (now-removed) in-process attempt. The 32 {@code long} fields exist purely so a given number + * of megabytes needs far fewer entries to reach {@code ObjectSampler}'s real 256KiB sampling + * floor - deliberately plain fields, not a nested array (a same-instance companion array would + * be a *second*, separate heap allocation the size-weighted allocation sampler would compete + * for, taking sampling attention away from {@code CachedPayload} itself). + */ + static final class CachedPayload { + final String key; + long p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15; + long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31; + + CachedPayload(String key) { + this.key = key; + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java new file mode 100644 index 0000000000..447a35cfbf --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java @@ -0,0 +1,216 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.JfrEvent; +import com.datadoghq.profiler.JfrEvents; +import org.openjdk.jmc.common.IMCType; +import org.openjdk.jmc.common.item.IAccessorKey; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.IType; +import org.openjdk.jmc.common.unit.IQuantity; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Shared {@code datadog.ReferenceChain} JFR-parsing helpers, extracted out of + * {@code ReferenceChainTrackingTest} so both that in-process JUnit test and + * {@link LeakingCacheScenario} (run inside a genuinely separate child JVM by + * {@code ExternalProcessReferenceChainTest}) can reuse the exact same JMC-accessor logic + * rather than maintaining two copies. + * + *

    {@link #findMatchForClass(JfrEvents, Class)} is a separate, jafar-backed counterpart to + * {@link #findMatchForClass(IItemCollection, Class)} for {@code ReferenceChainTrackingTest}, which + * loads recordings via {@code AbstractProfilerTest}'s {@code JfrEvents}-returning helpers. + * {@link LeakingCacheScenario} and {@code ReferenceChainJfrParserTest} + * load recordings directly via JMC's {@code JfrLoaderToolkit} and keep using the + * {@code IItemCollection} overload unchanged. + */ +public final class ReferenceChainAssertions { + private ReferenceChainAssertions() {} + + /** Result of {@link #findMatchForClass(IItemCollection, Class)}: one resolved chain event's fields. */ + public static final class ChainMatch { + public final List chain; + public final long targetTag; + public final int depth; + + ChainMatch(List chain, long targetTag, int depth) { + this.chain = chain; + this.targetTag = targetTag; + this.depth = depth; + } + } + + /** Result of {@link #findMatchForClass(JfrEvents, Class)}: one resolved chain event's fields. */ + public static final class JfrChainMatch { + public final List chain; + public final long targetTag; + public final int depth; + + JfrChainMatch(List chain, long targetTag, int depth) { + this.chain = chain; + this.targetTag = targetTag; + this.depth = depth; + } + } + + /** + * Scans {@code events} for a {@code datadog.ReferenceChain} item whose {@code chain[0]} is + * {@code targetClass} specifically, ignoring any events for other klasses this same + * leak-candidate mechanism may have legitimately flagged (e.g. "[B"/byte[] - see each caller's + * own comment). Returns {@code null} if {@code events} is empty or none match. + */ + public static ChainMatch findMatchForClass(IItemCollection events, Class targetClass) { + if (events == null || !events.hasItems()) { + return null; + } + for (IItemIterable iterable : events) { + IType type = iterable.getType(); + IMemberAccessor chainAccessor = findAccessor(type, "chain"); + IMemberAccessor targetTagAccessor = findAccessor(type, "targetTag"); + IMemberAccessor depthAccessor = findAccessor(type, "depth"); + if (chainAccessor == null) { + throw new IllegalStateException("No accessor for 'chain' field on datadog.ReferenceChain"); + } + + String targetName = jmcStyleName(targetClass); + for (IItem item : iterable) { + Object chainValue = chainAccessor.getMember(item); + if (!(chainValue instanceof Object[])) { + throw new IllegalStateException( + "'chain' field resolved to " + chainValue + ", expected an array"); + } + Object[] rawChain = (Object[]) chainValue; + if (rawChain.length == 0 || !(rawChain[0] instanceof IMCType) + || !targetName.equals(((IMCType) rawChain[0]).getFullName())) { + continue; + } + List chain = new ArrayList<>(rawChain.length); + for (Object element : rawChain) { + chain.add((IMCType) element); + } + long targetTag = targetTagAccessor != null ? numberValue(targetTagAccessor.getMember(item)) : -1; + int depth = depthAccessor != null ? (int) numberValue(depthAccessor.getMember(item)) : -1; + return new ChainMatch(chain, targetTag, depth); + } + } + return null; + } + + /** + * jafar/{@code JfrEvents}-backed counterpart to {@link #findMatchForClass(IItemCollection, Class)} - + * see this class's own header comment for why these are two separate overloads rather than one. + */ + public static JfrChainMatch findMatchForClass(JfrEvents events, Class targetClass) { + if (events == null || !events.hasItems()) { + return null; + } + String targetName = jmcStyleName(targetClass); + for (JfrEvent item : events) { + Object chainValue = item.get("chain"); + if (!(chainValue instanceof Object[])) { + throw new IllegalStateException( + "'chain' field resolved to " + chainValue + ", expected an array"); + } + Object[] rawChain = (Object[]) chainValue; + if (rawChain.length == 0 || !targetName.equals(classFullName(rawChain[0]))) { + continue; + } + List chain = new ArrayList<>(rawChain.length); + for (Object element : rawChain) { + chain.add(classFullName(element)); + } + long targetTag = item.getLong("targetTag", -1); + int depth = (int) item.getLong("depth", -1); + return new JfrChainMatch(chain, targetTag, depth); + } + return null; + } + + /** + * {@code targetClass}'s name in the same format {@code IMCType.getFullName()} uses for array + * types - found the hard way: {@code Class.getName()} renders {@code byte[].class} as {@code + * "[B"} (JVM internal signature notation), but JMC's chunk parser renders the same array class's + * {@code IMCType.getFullName()} as {@code "byte[]"} (Java source notation), so comparing {@link + * Class#getName()} directly against {@code getFullName()} - as both {@code findMatchForClass} + * overloads above used to - can never match an array-typed leaf class, even when the correct + * event is genuinely present in the recording. Only affects array {@code targetClass} values + * (e.g. {@code byte[].class}); a non-array class's {@code getName()} already matches {@code + * getFullName()} as-is. + */ + private static String jmcStyleName(Class targetClass) { + int dimensions = 0; + Class component = targetClass; + while (component.isArray()) { + dimensions++; + component = component.getComponentType(); + } + if (dimensions == 0) { + return targetClass.getName(); + } + StringBuilder name = new StringBuilder(component.getName()); + for (int i = 0; i < dimensions; i++) { + name.append("[]"); + } + return name.toString(); + } + + /** + * The full name (e.g. {@code java.lang.String}) of a resolved {@code chain[]} array element - + * mirrors {@code JfrEvent.getClassName(String)}'s own class-reference-map unwrapping, applied + * to an array element rather than a named field. + */ + @SuppressWarnings("unchecked") + private static String classFullName(Object element) { + if (!(element instanceof Map)) { + throw new IllegalStateException( + "chain[] element resolved to " + element + ", expected a class reference map"); + } + Object name = ((Map) element).get("name"); + String s; + if (name instanceof Map) { + Object v = ((Map) name).get("string"); + s = v != null ? v.toString() : null; + } else { + s = name != null ? name.toString() : null; + } + return s != null ? s.replace('/', '.') : null; + } + + /** + * Looks up a field's accessor by identifier rather than via {@code Attribute.attr(...)}: JMC's + * v1 chunk parser (internal.parser.v1.ValueReaders.ArrayReader#getContentType()) registers + * {@code UnitLookup.UNKNOWN} as the declared content type for every array field regardless of + * what its element reader resolves to, so an {@code F_ARRAY} field like {@code chain} + * (T_CLASS, F_CPOOL|F_ARRAY, jfrMetadata.cpp) cannot be bound via a compile-time-typed + * {@code Attribute}. Mirrors {@code ReferenceChainJfrParserTest}'s identical lookup, which + * already proves this resolves {@code chain}'s array elements to real {@link IMCType}s. + */ + public static IMemberAccessor findAccessor(IType type, String identifier) { + for (IAccessorKey key : type.getAccessorKeys().keySet()) { + if (identifier.equals(key.getIdentifier())) { + return type.getAccessor(key); + } + } + return null; + } + + public static long numberValue(Object value) { + if (value instanceof Number) { + return ((Number) value).longValue(); + } + if (value instanceof IQuantity) { + return ((IQuantity) value).longValue(); + } + return -1; + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java new file mode 100644 index 0000000000..d33072a588 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java @@ -0,0 +1,192 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import org.junit.jupiter.api.Test; +import org.openjdk.jmc.common.IMCType; +import org.openjdk.jmc.common.item.IAccessorKey; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.IType; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.flightrecorder.CouldNotLoadRecordingException; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * PROF-15341 design doc, Open Question: does JMC's parser actually resolve the {@code + * datadog.ReferenceChain} event's {@code chain} field - declared in jfrMetadata.cpp as {@code + * field("chain", T_CLASS, ..., F_CPOOL | F_ARRAY)}, i.e. an array of scalar + * constant-pool-index {@code T_CLASS} values - the same way it resolves a plain scalar + * F_CPOOL field (e.g. {@code objectClass}, already exercised by {@code AbstractProfilerTest}) + * or a plain F_ARRAY-of-composite-struct field (e.g. {@code jdk.StackTrace#frames})? Neither + * already-exercised shape proves this exact combination. + * + *

    This test loads the standalone .jfr file produced by the companion gtest ({@code + * ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp}) directly via {@link + * JfrLoaderToolkit} - no live profiler attach, no {@code AbstractProfilerTest} lifecycle - and + * asserts the {@code chain} field resolves to the exact class names that gtest seeded, in the + * exact leaf-to-root order {@code ReferenceChainTracker::buildChainEvent()} (referenceChains.h) + * produces. + * + *

    Why a generic accessor, not {@code Attribute.attr(...)}: JMC's v1 chunk parser + * (internal.parser.v1.ValueReaders.ArrayReader#getContentType()) registers {@code + * UnitLookup.UNKNOWN} as the declared content type for every array field, regardless of + * what its element reader resolves to - this is generic to all array fields, not specific to the + * cpool case. A field registered as UNKNOWN cannot be bound via {@code + * Attribute.attr(id, name, desc, CLASS).getAccessor(type)} (that requires the field's registered + * content type to match). This test instead looks the field up by identifier via {@code + * IType#getAccessorKeys()} and calls {@code IType#getAccessor(IAccessorKey)} directly - the same + * lower-level lookup JMC's own UI uses for fields it has no compile-time-known attribute for. + * This is not a workaround for a missing capability; it is the correct API for an unregistered + * custom field, and it still calls through the very code + * ({@code ArrayReader.read()}/{@code resolve()} delegating per-element to the field's element + * reader, which for {@code chain} is a {@code PoolReader}) that resolves each array element's + * constant-pool index to its class - the actual thing this test exists to prove. + */ +public class ReferenceChainJfrParserTest { + + private static final String EVENT_TYPE = "datadog.ReferenceChain"; + + /** + * Same path {@code chainRoundtripJfrPath()} in the companion gtest resolves to: the OS temp + * dir (TMPDIR, falling back to /tmp), agreed by both sides rather than by a shared build + * directory - the gtest (ddprof-lib) and this test (ddprof-test) are different Gradle + * modules/tasks with no other filesystem contract between them. + */ + private static Path roundtripJfrPath() { + String tmp = System.getenv("TMPDIR"); + String dir = (tmp != null && !tmp.isEmpty()) ? tmp : "/tmp"; + return Paths.get(dir, "datadog_reference_chain_roundtrip.jfr"); + } + + @Test + public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() + throws IOException, CouldNotLoadRecordingException { + Path jfrPath = roundtripJfrPath(); + assertTrue(Files.exists(jfrPath), + "Expected " + jfrPath + " to exist - run " + + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " + + "(referenceChainJfrRoundtrip_ut.cpp produces this file)."); + + IItemCollection events; + try (InputStream in = Files.newInputStream(jfrPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + IItemCollection chainEvents = events.apply(ItemFilters.type(EVENT_TYPE)); + assertTrue(chainEvents.hasItems(), "Expected at least one " + EVENT_TYPE + " event"); + + List resolvedChain = null; + long targetTag = -1; + int depth = -1; + for (IItemIterable iterable : chainEvents) { + IType type = iterable.getType(); + IMemberAccessor chainAccessor = findAccessor(type, "chain"); + IMemberAccessor edgesAccessor = findAccessor(type, "edges"); + IMemberAccessor targetTagAccessor = findAccessor(type, "targetTag"); + IMemberAccessor depthAccessor = findAccessor(type, "depth"); + assertNotNull(chainAccessor, "No accessor for 'chain' field on " + EVENT_TYPE); + assertNotNull(edgesAccessor, "No accessor for 'edges' field on " + EVENT_TYPE + + " - the new retention-edge label array (T_STRING|F_ARRAY) did not make it " + + "into the recorded metadata"); + + for (IItem item : iterable) { + Object chainValue = chainAccessor.getMember(item); + assertNotNull(chainValue, "'chain' field resolved to null"); + assertTrue(chainValue instanceof Object[], + "'chain' field resolved to " + chainValue.getClass() + ", expected an array"); + + Object[] rawChain = (Object[]) chainValue; + List chain = new ArrayList<>(rawChain.length); + for (Object element : rawChain) { + assertNotNull(element, + "chain[] element resolved to null - the constant-pool reference for this " + + "T_CLASS array entry did not resolve to a class"); + assertTrue(element instanceof IMCType, + "chain[] element resolved to " + element.getClass() + + ", expected " + IMCType.class + " (a resolved class, not a raw cpool index)"); + chain.add((IMCType) element); + } + resolvedChain = chain; + // Retention-edge labels: one per chain hop, aligned with it (the + // gtest seeds kind labels because its JNIEnv is null - the label + // DECODE is skipped; the Java side validates the ARRAY FIELD parses, + // not the decode itself, which referenceChains_ut covers natively). + Object edgesValue = edgesAccessor.getMember(item); + assertNotNull(edgesValue, "'edges' field resolved to null"); + assertTrue(edgesValue instanceof Object[], + "'edges' field resolved to " + edgesValue.getClass() + + ", expected an array"); + Object[] rawEdges = (Object[]) edgesValue; + assertEquals(3, rawEdges.length, "edges must align with the chain, one label per hop"); + for (Object e : rawEdges) { + assertTrue(e instanceof String, + "edges[] element resolved to " + (e == null ? "null" : e.getClass()) + + ", expected a String"); + } + assertEquals("field", rawEdges[0]); + assertEquals("element", rawEdges[1]); + assertEquals("jni_global", rawEdges[2]); + if (targetTagAccessor != null) { + Object v = targetTagAccessor.getMember(item); + if (v instanceof Number) { + targetTag = ((Number) v).longValue(); + } else if (v instanceof org.openjdk.jmc.common.unit.IQuantity) { + targetTag = ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); + } + } + if (depthAccessor != null) { + Object v = depthAccessor.getMember(item); + if (v instanceof Number) { + depth = ((Number) v).intValue(); + } else if (v instanceof org.openjdk.jmc.common.unit.IQuantity) { + depth = (int) ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); + } + } + break; // referenceChainJfrRoundtrip_ut.cpp writes exactly one event + } + if (resolvedChain != null) { + break; + } + } + + assertNotNull(resolvedChain, "Never iterated a " + EVENT_TYPE + " item"); + assertEquals(3, resolvedChain.size(), "Expected leaf/middle/root - 3 entries"); + // Leaf-to-root order, matching ReferenceChainTracker::buildChainEvent()'s + // FrontierTable::reconstructChain() contract and the gtest's insert() calls + // (tag=3 leaf -> tag=2 middle -> tag=1 root). + assertEquals("com.test.ChainLeaf", resolvedChain.get(0).getFullName()); + assertEquals("com.test.ChainMiddle", resolvedChain.get(1).getFullName()); + assertEquals("com.test.ChainRoot", resolvedChain.get(2).getFullName()); + assertEquals(3L, targetTag, "targetTag should be the leaf's frontier tag (3)"); + assertEquals(2, depth, "depth should be the leaf's own depth (2 hops from the root)"); + } + + private static IMemberAccessor findAccessor(IType type, String identifier) { + Map, ?> keys = type.getAccessorKeys(); + for (IAccessorKey key : keys.keySet()) { + if (identifier.equals(key.getIdentifier())) { + return type.getAccessor(key); + } + } + return null; + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java new file mode 100644 index 0000000000..0d33e8fba2 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java @@ -0,0 +1,220 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * PROF-15341 follow-up: {@code ReferenceChainTrackingTest} exercises {@code LivenessTracker}'s + * probabilistic allocation-sampling-driven slope detection and {@code ReferenceChainTracker}'s + * root-seeded BFS discovery together, in one real-JVM run - reliable only when both mechanisms + * happen to line up within the same bounded retry window. This class decouples them via the + * debug-build-only test seams on {@link JavaProfiler} (backed by {@code LivenessTracker}'s + * existing {@code *ForTest} seams and a new {@code ReferenceChainTracker::tagAsRootForTest()}, + * see javaApi.cpp), so each mechanism can be verified end-to-end in isolation: + *

      + *
    • {@link #shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope()} - asserts a slope signal + * would fire from directly-seeded population history, with no allocation sampling involved.
      • + *
    • {@link #shouldReconstructChainForDirectlyTaggedRoot()} - asserts the BFS/chain- + * reconstruction/event-emission path fires for a directly-tagged, known live object, with no + * dependency on {@code selectLeakCandidates()} organically picking the right klass.
      • + *
    + * + *

    Only runs under the debug native build ({@code testdebug}) - the backing native methods do + * not exist in a release build (see javaApi.cpp's {@code #ifdef DEBUG} guard), mirroring + * {@code JVMAccessTest}'s own {@code "debug".equals(System.getProperty("ddprof_test.config"))} + * pattern for the same reason. + */ +public class ReferenceChainTestSeamsTest extends AbstractProfilerTest { + + // Arbitrary, test-chosen klass ids - opaque keys for the population table, EXCEPT + // where a representative is involved: setKlassPopulationRepresentativeForTest() now + // aliases the representative's REAL class id onto the seeded synthetic id (the + // leak-tag redesign keys every consumer by real ids), so shouldReconstructChainForDirectlyTaggedRoot() + // must target a class whose real population entry does not already exist - a unique + // nested class, not java.lang.Object - otherwise the real entry's genuine (noisy) + // history outranks the seeded ramp and the target never becomes a candidate. + private static final int SLOPE_TEST_KLASS_ID = 987001; + private static final int CHAIN_TEST_KLASS_ID = 987002; + + /** Unique class for {@link #shouldReconstructChainForDirectlyTaggedRoot()}'s target. */ + private static final class ChainTarget {} + + /** + * Durable root for {@link #shouldReconstructChainForDirectlyTaggedRoot()}'s target: the + * noise gate (suppressChainEvent) intentionally drops depth<2 chains whose root is + * transient (stack local / JNI local) - a method-local target is transient BY DESIGN, so + * holding it statically (the canonical leak shape: static field) is required for the + * chain event to be emitted at all. Shared-JVM hygiene: nulled in a finally below so the + * next test in this no-forkEvery JVM does not inherit the reference. + */ + private static Object chainTargetHolder; + + @Override + protected String getProfilerCommand() { + // generations=true: gates LivenessTracker's population tracking (gcGenerationsEnabled()) - + // required for selectLeakCandidates() to return anything at all, real or seeded. + // referencechains=true: constructs ReferenceChainTracker's FrontierTable so + // tagAsReferenceChainRoot0()/runReferenceChainPass0() have a table to insert into. + // framecap=2000000 (not 1024): shouldReconstructChainForDirectlyTaggedRoot()'s first pass + // runs IterateOverReachableObjects, which admits every GC root in the whole JVM - not just + // this test's directly-tagged target - into the same frontier. A small framecap fills with + // that ambient root count before the tagged root ever gets to expandFrontier(), hitting + // SearchAbandonReason::FRONTIER_CAP with no chain event queued. Same value/rationale as + // ReferenceChainTrackingTest's own success-path tests (see that class's header comment). + return "generations=true,referencechains=true:hops=32:budget=500:ttl=60000:framecap=2000000"; + } + + @Override + protected boolean isPlatformSupported() { + return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); + } + + private static void assumeDebugBuild() { + assumeTrue("debug".equals(System.getProperty("ddprof_test.config"))); + } + + /** + * Seeds twenty epochs of strictly increasing population counts for {@link #SLOPE_TEST_KLASS_ID} + * directly into LivenessTracker's ring buffer - bypassing real allocation sampling entirely - + * then asserts {@code selectLeakCandidates()} ranks it as a leak candidate. This is the "assert + * a slope signal would be generated" seam: it proves the ranking logic itself works without + * depending on the real JVMTI heap sampler ever surfacing this specific klass. + */ + @Test + public void shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope() { + assumeDebugBuild(); + JavaProfiler.resetKlassPopulationForTest0(); + JavaProfiler.setGcGenerationsEnabled0(true); + + // KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 (livenessTracker.h) is only the floor at which a + // trend becomes eligible at all - selectLeakCandidates() also requires consecutive_positive + // to reach LEAK_TREND_HYSTERESIS_BASE = 5 consecutive qualifying epochs before trusting it + // (the sustained-trend hysteresis gate), so at least 10 + (5 - 1) = 14 strictly increasing + // samples are needed; 20 gives headroom. + for (int epoch = 1; epoch <= 20; epoch++) { + JavaProfiler.seedKlassPopulationSample0(SLOPE_TEST_KLASS_ID, epoch * 10, epoch); + // Per-(klass, tid) qualification: selectLeakCandidates() now requires a + // qualifying allocating thread on top of the klass-level ramp. This test + // only asserts the klass makes it into the candidate list - it has no + // tracked instances whose tid a tagging scope would need to match - so a + // fixed synthetic tid suffices here. + JavaProfiler.seedTidTrendSample0(SLOPE_TEST_KLASS_ID, 4242, epoch * 3, epoch); + } + + int[] candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + boolean found = false; + for (int klassId : candidates) { + if (klassId == SLOPE_TEST_KLASS_ID) { + found = true; + break; + } + } + assertTrue(found, "Expected klass id " + SLOPE_TEST_KLASS_ID + + " to be selected as a leak candidate after a seeded positive-slope population history"); + } + + /** + * Tags a real, live, caller-chosen object directly as a reference-chain frontier root + * (bypassing ReferenceChainTracker's normal root-seeded discovery walk), wires it in as a + * seeded leak candidate's representative, then drives one BFS pass and one poll cycle + * synchronously. This is the "trigger the refchain on a known live heap sample" seam: it + * proves {@code runPass()}/{@code pollWatchedTargets()}/{@code buildChainEvent()} correctly + * produce a chain event for a target this test controls directly, decoupled from whether + * LivenessTracker's probabilistic sampler would have picked the same object on its own. + */ + @Test + public void shouldReconstructChainForDirectlyTaggedRoot() { + assumeDebugBuild(); + JavaProfiler.resetKlassPopulationForTest0(); + JavaProfiler.setGcGenerationsEnabled0(true); + // Guards against inheriting a FrontierTable an earlier test in this same, no-forkEvery + // JVM left permanently full/tiny - e.g. ReferenceChainTrackingTest's own + // shouldReportAbandonedSearchOnTinyFrontierCap deliberately drives the shared table to + // framecap=1 and leaves it that way. Same defensive pattern that class's own tests already + // use (see its header comment); without it, tagAsReferenceChainRoot0()'s insert() below can + // fail against a table this test never sized itself. + JavaProfiler.resetReferenceChainSearchForTest0(); + + // The target is referenced ONLY through the static holder - never bound to a + // live local of this frame. A local `target` variable would make the object a + // STACK_LOCAL GC root for the whole test body, and the noise gate (rightly) + // suppresses depth-0 chains rooted in a transient stack slot: the durable + // roots this fixture wants (the static holder, plus the JNI-global-weak-ref + // root LivenessTracker's representative itself creates) then lose the + // durability race only if a stack-local root exists at all. Accessing the + // object only via the static field leaves its frame slots empty between + // calls, so no stack root is ever enumerated. + chainTargetHolder = new ChainTarget(); + + // Representative BEFORE seeding: with the aliasing seam, setting the representative is + // what registers the synthetic->real id alias, so the seeding below lands re-keyed under + // the target's real class id (seeding first still works - the synthetic entry gets + // re-keyed at representative-set time - but rep-first is the documented load-bearing order). + JavaProfiler.setKlassPopulationRepresentativeForTest0(CHAIN_TEST_KLASS_ID, chainTargetHolder); + + // See shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope()'s comment above for why 20 + // (not just KLASS_POPULATION_MIN_FILL_FOR_TREND = 10) is needed to clear the hysteresis gate. + for (int epoch = 1; epoch <= 20; epoch++) { + JavaProfiler.seedKlassPopulationSample0(CHAIN_TEST_KLASS_ID, epoch * 10, epoch); + // Per-(klass, tid) qualification, seeded with THIS thread's real + // profiler tid: the target was allocated here, so if any path in the + // pass/poll cycle below does hinge on leak tagging of the target as a + // tracked instance, the qualifying tid matches it. (The chain's primary + // path is the directly-tagged frontier root + seeded representative, so + // this is belt-and-braces, not a proven requirement.) + JavaProfiler.seedTidTrendSample0( + CHAIN_TEST_KLASS_ID, JavaProfiler.getTid(), epoch * 3, epoch); + } + + // One poll BEFORE the direct tag: the first pollReferenceChainTargets0() + // admits the seeded klass as a watched candidate (canary slot), and the + // seam's discovery recording (tagAsRootForTest's recordDiscoveredInstance) + // only lands when a candidate slot already watches the klass - the + // leak-tag redesign removed the marker-tag path the original pre-poll tag + // used to ride. + JavaProfiler.pollReferenceChainTargets0(); + + // The target is referenced ONLY through the static holder - never bound to a + // live local of this frame. A local `target` variable would make the object a + // STACK_LOCAL GC root for the whole test body, and the noise gate (rightly) + // suppresses depth-0 chains rooted in a transient stack slot: the durable + // roots this fixture wants (the static holder, plus the JNI-global-weak-ref + // root LivenessTracker's representative itself creates) then lose the + // durability race only if a stack-local root exists at all. Accessing the + // object only via the static field leaves its frame slots empty between + // calls, so no stack root is ever enumerated. + long tag = JavaProfiler.tagAsReferenceChainRoot0(chainTargetHolder); + assertTrue(tag > 0, "Expected tagAsReferenceChainRoot0 to assign a valid frontier tag"); + + // Drive pass+poll cycles until the chain event lands. One pass is NOT enough: + // admitStaticFieldRoots() sweeps loaded classes in chunks (a few hundred per pass, + // bounded by the budget) and the first pass truncates before reaching this test's + // class - the durable (static-field) root discovery that upgrades the target's + // root_kind only happens on the pass whose chunk covers ChainTarget's holder class. + // The bound (40) gives the full sweep lap (~10 passes for ~4000 loaded classes) + // several round trips of headroom; the BFS thread is excluded from racing these + // calls by the engine lock (runPassSerialized/pollWatchedTargetsSerialized). + int eventCount = 0; + for (int pass = 0; pass < 40 && eventCount == 0; pass++) { + boolean sawPassRun = JavaProfiler.runReferenceChainPass0(); + assertTrue(sawPassRun || pass > 0, + "Expected the first runReferenceChainPass0 to run (reference chains enabled)"); + JavaProfiler.pollReferenceChainTargets0(); + eventCount = JavaProfiler.drainReferenceChainEventCount0(); + } + assertTrue(eventCount > 0, + "Expected pollWatchedTargets() to have queued at least one chain event for the " + + "directly-tagged, seeded-representative target"); + chainTargetHolder = null; // shared-JVM hygiene + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java new file mode 100644 index 0000000000..c3fc86ec6b --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java @@ -0,0 +1,979 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.JfrEvent; +import com.datadoghq.profiler.JfrEvents; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junitpioneer.jupiter.RetryingTest; + +import java.nio.file.Files; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicLong; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * PROF-15341 (+ lifecycle-wiring follow-up, + the Remaining Work Plan's target-selection bridging, + * pause-time pacing, and reporting work): end-to-end + * coverage for {@code ReferenceChainTracker} (ddprof-lib/src/main/cpp/referenceChains.h/.cpp). + * + *

    Scope note: {@code ReferenceChainTracker::start()} is now called from + * {@code Profiler::start()} (profiler.cpp), gated on {@code args._reference_chains}, followed by + * {@code ReferenceChainTracker::startThread()} which spawns its BFS thread; {@code Profiler::stop()} + * calls the matching {@code stopThread()}/{@code stop()} pair. A {@code datadog.ReferenceChainAbandoned} + * event now reaches a live {@code Recording}: {@code Profiler::dump()} calls + * {@code buildAbandonedEvent()} and writes its output via + * {@code Profiler::writeReferenceChainAbandoned()}/{@code FlightRecorder::recordReferenceChainAbandoned()} + * whenever the search has ended in {@code SearchState::ABANDONED} - mirroring the + * {@code LivenessTracker::flush()} call site there. {@link #shouldReportAbandonedSearchOnTinyFrontierCap()} + * exercises that path end-to-end below. + * + *

    {@code buildChainEvent()}'s target-selection feed (Remaining Work Plan's target-selection + * bridging step): + * {@code ReferenceChainTracker::pollWatchedTargets()} (referenceChains.cpp), called from + * {@code threadLoop()} once per scheduling cycle after {@code runPass()}, closes the gap this class's + * header comment used to describe as unclosed. It polls + * {@code LivenessTracker::selectLeakCandidates()} (positive population-slope ranking over a rolling + * per-klass survivor-count window, gated on {@code _gc_generations} - design doc's Open Question 3) + * and, for each ranked klass whose representative instance an ordinary {@code runPass()} walk has + * *already* tagged ({@code getTag() > 0} - a read, never a {@code SetTag} seed), reconstructs and + * emits its chain via {@code Profiler::writeReferenceChain()}. {@link #shouldReconstructReferrerChainToGcRoot()} + * below exercises this end to end against a real JVM, real GCs, and a real JVMTI heap walk - not a + * synthetic frontier fixture (see {@code referenceChainJfrRoundtrip_ut.cpp}/{@code ReferenceChainJfrParserTest} + * for that already-covered, lower-level reconstruction-correctness proof). + * + *

    Why {@code @TestMethodOrder}/{@code @Order(1)}: {@code ReferenceChainTracker} is a + * process-wide singleton whose single, singleton-owned search (design doc's Open Question 3 + * "Shipped" note) never restarts once it leaves {@code SearchState::RUNNING} - + * {@code shouldRunPass()} (referenceChains.cpp) returns {@code false} forever after that point, and + * the transition itself releases every tag the search ever assigned + * ({@code releaseSearchTags()}). Its {@code FrontierTable} is sized once, the first time any test + * in this JVM calls {@code ReferenceChainTracker::start()}, and never resized on later + * start()/stop() cycles (referenceChains.cpp's {@code if (_frontier == nullptr)} guard) - so + * whichever test runs first also fixes that capacity for every test that runs after it in the same + * JVM (no {@code forkEvery} configured, see {@code ProfilerTestPlugin.kt}, so this whole class runs + * in one). {@link #shouldReportAbandonedSearchOnTinyFrontierCap()} below deliberately drives that + * shared search to {@code SearchState::ABANDONED}, via an artificially tiny frontier cap when it + * gets to build the table itself, or its own {@code ttl} fallback otherwise (see that method's own + * comment). Both {@link #shouldReconstructReferrerChainToGcRoot()} and + * {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()} need the search still + * {@code RUNNING} to find their own candidates' tags non-zero, so they are pinned to run first, + * via {@code @Order(1)}/{@code @Order(2)} respectively - without that ordering all three tests + * would race for which ones get to observe a still-{@code RUNNING} search, and neither + * success-path test has a fallback for losing that race the way the abandonment test does. The + * relative order between the two success-path tests does not itself matter - both target + * different klasses ({@link ChainLink} vs {@link CachedPayload}) within the same shared search, + * and neither exhausts it - only their both running before the abandonment test does. + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@Tag("slow") +public class ReferenceChainTrackingTest extends AbstractProfilerTest { + + // Arbitrary, test-chosen klass ids for the debug-only population-seeding seams below (see + // ReferenceChainTestSeamsTest's own comment: LivenessTracker's population table treats these as + // opaque keys, so they need not resolve to any real class). Distinct per test/from + // ReferenceChainTestSeamsTest's own ids purely as cheap insurance against collision in a shared, + // no-forkEvery test JVM. + private static final int CHAIN_LINK_TEST_KLASS_ID = 987201; + private static final int CACHED_PAYLOAD_TEST_KLASS_ID = 987202; + + // Durable (static-field) holder for the gc-root fixture below. The + // discovered-chain gate suppresses chains shallower than the first real + // holder hop that are rooted at a TRANSIENT root (stack/JNI local) - a + // frame-held "leak" is by definition not a leak - and + // shouldReconstructReferrerChainToGcRoot()'s ChainLink instances sit at + // depth 1 of a locally-held list, exactly the transient-rooted shape the + // gate exists to suppress. A static field gives them the real + // unmaintained-singleton-collection retention shape instead. (The cache + // fixture below needs no such change: its CachedPayload chains are three + // hops deep through HashMap's own internals, which the gate correctly + // lets through regardless of root kind.) Cleared in the test's finally: + // this is a shared, no-forkEvery test JVM, and a leftover holder would + // keep its entire population reachable for every later test here. + private static List gcRootHolder; + + // The current round's allocation batch, handed to the persistent allocator + // thread below. Volatile: written here, read by the allocator thread every + // round - the request queue's happens-before (offer after set, take before + // read) already covers it, but the visibility of a plain field handed + // across threads is kept explicit rather than relying on queue semantics + // alone. + private static volatile List currentBatch; + + // LivenessTracker's own hysteresis gate (livenessTracker.h's + // KLASS_POPULATION_MIN_FILL_FOR_TREND=10 / LEAK_TREND_HYSTERESIS_BASE=5; + // livenessTracker.cpp's recordKlassPopulationSampleLocked()/hasQualifyingGrowth()) only starts + // incrementing KlassPopulationEntry::consecutive_positive once ring_fill has reached 10 - every + // call before that resets it straight back to 0 (ringThirdsStats()'s own fill < min_fill guard). + // A single batch of exactly 10 seedKlassPopulationSample0() calls therefore caps + // consecutive_positive at 1 (only the 10th call ever sees ring_fill==10), far short of the >=5 + // (or >=3 if LivenessTracker::heapFloorRising() independently corroborates) that + // selectLeakCandidates() (livenessTracker.cpp) requires before it reports a candidate at all - + // which is what ReferenceChainTracker::hasLeakSignal() consults, and what shouldRunPass() + // (referenceChains.cpp) gates even this search's very first pass on whenever generations=true + // (see this class's own header comment on why runPass() never otherwise fires here without one). + // 10 (KLASS_POPULATION_MIN_FILL_FOR_TREND) + 5 (LEAK_TREND_HYSTERESIS_BASE) - 1 = 14 calls are + // the minimum for the 5 qualifying calls at ring_fill>=10 (epochs 10-14) to actually land; +1 + // for margin against off-by-one uncertainty in that derivation. + private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; + + // Dedicated id for shouldReportAbandonedSearchOnTinyFrontierCap()'s own seeding (see that + // method's own comment) - distinct from CHAIN_LINK_TEST_KLASS_ID/CACHED_PAYLOAD_TEST_KLASS_ID + // for the same cheap-insurance-against-collision reason those two are distinct from each other. + private static final int ABANDON_TEST_KLASS_ID = 987203; + + @Override + protected String getProfilerCommand() { + String testName = testInfo != null + ? testInfo.getTestMethod().map(java.lang.reflect.Method::getName).orElse("") + : ""; + if ("shouldReconstructReferrerChainToGcRoot".equals(testName) + || "shouldReconstructReferrerChainThroughUnboundedCacheLeak".equals(testName)) { + // memory=...:l + generations=true: LivenessTracker::gcGenerationsEnabled() (Remaining + // Work Plan's population tracking / target-selection bridging, livenessTracker.h) must be true for pollWatchedTargets() + // (referenceChains.cpp) to do anything at all - referencechains=... alone stays + // whole-graph-only (design doc's Open Question 3 "still undecided" fallback). A small + // sampling interval maximizes the chance the many ChainLink instances this test + // allocates below actually get liveness-tracked (LivenessTracker::track(), invoked from + // the allocation-sampling path whenever _gc_generations || _record_liveness is set, + // objectSampler.cpp). A large framecap avoids an early + // SearchAbandonReason::FRONTIER_CAP abandonment against this JVM's real, + // not-controlled-by-this-test root-reachable graph size, and a long ttl avoids a + // TTL abandonment mid-test - either would call releaseSearchTags() and permanently + // zero every tag this test depends on (see this class's header comment on why this + // test is pinned to @Order(1)). + // + // pausetarget=5000: the pause-time pacing controller's updatePacing() (referenceChains.cpp) measures this JVM's real + // FollowReferences/GetObjectsWithTags call latency at ~150ms early on, climbing past a full + // second as this test's own allocation keeps handing the search more to discover (each pass + // re-scans the whole already-discovered/tagged set, so cost grows with cumulative progress, + // not with this test's own allocation *rate*) - i.e. that latency is dominated by per-call + // overhead, not by the requested edge budget. Against the default pausetarget=5ms ceiling + // (DEFAULT_REFERENCE_CHAINS_PAUSE_TARGET_MS, arguments.h) every observed pass is "over + // ceiling even at the floor", so the controller settles at MIN_EFFECTIVE_BUDGET=50 edges/pass + // and MAX_EFFECTIVE_CADENCE_NS=4s between passes - a real but glacial discovery rate this + // test's own wall-clock budget cannot outwait, and one a merely-generous ceiling (e.g. 500ms) + // only postpones: once cumulative progress pushes the real per-pass cost back past *that* + // ceiling too, the same throttling recurs. A ceiling comfortably above the highest per-call + // cost this test's own scale of population growth is expected to reach instead lets the + // controller keep _effective_budget at this test's own configured budget=4000 ceiling for + // the method's entire run, the same convergence behavior referenceChains_ut.cpp's own pacing + // tests already exercise synthetically. + // + // painbudget=100: shouldReconstructReferrerChainThroughUnboundedCacheLeak shares this + // singleton search with shouldReconstructReferrerChainToGcRoot (@Order(1), runs first) and + // needs a restart (canAffordNewSearch(), referenceChains.cpp) once that first search + // reaches a terminal state. PainBudget (painBudget.h) gates that restart on + // pain_spent_ms / refill_rate milliseconds of cooldown; the default + // DEFAULT_REFERENCE_CHAINS_PAIN_BUDGET_PERCENT=1 (1% refill) can make that cooldown run to + // tens of seconds to minutes for a search this large - far past this test's own bounded + // retry loop (16 rounds plus a short grace period). painbudget=100 keeps the restart gate + // affordable immediately, which is fine here since nothing else in this JVM is competing + // for the pain budget. + // + // firstpassbudget=200000: root/stack-ref enumeration (runPassManualWalk()'s + // IterateOverReachableObjects call) shares budget=4000 with expandFrontier()'s steady-state + // expansion unless overridden - in this shared, forked test JVM's real (not controlled by + // this test) root-reachable graph, that leaves this method's own CachedPayload-holding + // cache HashMap (a direct stack-local root on this test's own thread) at the mercy of + // whatever order JVMTI's enumeration happens to visit roots in: budget=4000 was observed + // exhausting itself on other roots' subtrees every single attempt, never once admitting a + // single edge from this thread's own stack. Overriding just root enumeration's own budget + // (not budget=4000, which the pacing controller has already been tuned around - raising it + // instead throttles every expansion pass down to + // MIN_EFFECTIVE_BUDGET/MAX_EFFECTIVE_CADENCE_NS, see updatePacing()) gives every root + // enumeration attempt (the first pass, and any later one ROOT_ENUM_MIN_INTERVAL_NS gates + // back in - see that constant's own comment) enough headroom to reach this thread's stack + // without slowing down the steady-state expansion passes that follow. + // memory=64:l:1.0 - the explicit 100% live-sample ratio is load-bearing: + // without a ratio segment the default is 10% (arguments.h's + // _live_samples_ratio) and LivenessTracker::track() drops 90% of + // tracked instances probabilistically, thinning the pool the + // representative-minting and candidate trend rings draw from - + // observed live as the intermittent "representative died/evicted" + // and zero-tag runs (see LeakTagCorrelationReferenceChainTest's own + // comment for the full lottery analysis). Total tracking volume is + // still capped by the 256KiB sampling-interval floor, not the ratio, + // so 100% keeps the tracking table tiny. + return "memory=64:l:1.0,generations=true," + + "referencechains=true:hops=64:budget=4000:ttl=120000:framecap=2000000:pausetarget=5000:painbudget=100:firstpassbudget=200000"; + } + // shouldReportAbandonedSearchOnTinyFrontierCap needs the frontier table (not the + // per-pass budget) to be what runs out first: heapReferenceCallback() (referenceChains.cpp) + // checks the budget before ever calling FrontierTable::insert(), so a budget as small as + // the frontier cap itself (e.g. budget=1:framecap=1) only ever exhausts the budget after + // admitting exactly one object - insert() never gets a chance to fail, and the search + // completes normally instead of abandoning. A budget comfortably larger than framecap=1 + // lets a *second* root-referenced object reach insert() and hit the actually-full table, + // which is what runPass() treats as grounds to abandon (Termination section priority 1). + // + // framecap=1 only actually controls the table's capacity the *first* time + // ReferenceChainTracker::start() ever constructs its FrontierTable in this JVM + // (referenceChains.cpp's `if (_frontier == nullptr)` guard - the table, like + // LivenessTracker's own, survives every later start()/stop() cycle at whatever size it was + // first built with). This method's own @Order(1) pinning on + // shouldReconstructReferrerChainToGcRoot means *that* test's own, much larger framecap=2000000 + // wins that race instead - so this method also supplies a small ttl as a second, independent + // abandonment trigger. runPass()'s Termination-section priority still checks frontier-cap + // first, so a fresh, genuinely-tiny table (this method running first, e.g. in isolation via + // `-Ptests=ReferenceChainTrackingTest.shouldReportAbandonedSearchOnTinyFrontierCap`) still + // abandons via SearchAbandonReason::FRONTIER_CAP as originally designed; inheriting an + // already-oversized table instead falls through to the ttl check, which fires almost + // immediately regardless of table size because _search_start_ns (referenceChains.h) is set + // once, the first time the shared search ever started - by the time this method's own + // start() call re-parses ttl, that clock already reads however long + // shouldReconstructReferrerChainToGcRoot's own run took. Either path produces a + // datadog.ReferenceChainAbandoned event, which is all this method actually asserts on. + // + // painbudget=100, for the same reason shouldReconstructReferrerChainThroughUnboundedCacheLeak() + // above needs it (see that method's own comment on painbudget=100): this method's own + // resetReferenceChainSearchForTest0() call spends whatever _search_pain_ms + // shouldReconstructReferrerChainThroughUnboundedCacheLeak()'s own long-running search left + // behind (found the hard way - that search runs 20+ passes admitting tens of thousands of + // edges each, and nothing zeroes _search_pain_ms between its own end and this method's start() + // reconstructing a fresh, zero-balance PainBudget) into *this* method's own PainBudget + // (referenceChains.cpp's restartSearch()/resetSearchStateForTest(), both call + // _pain_budget.spend(_search_pain_ms) unconditionally) - at the default 1% refill rate, + // draining that debt back to canStartNow()==true would take on the order of minutes, far past + // this method's own retry budget below. painbudget=100 keeps that gate affordable immediately, + // the same way it does for the sibling method. + if ("shouldReportAbandonedSearchOnTinyFrontierCap".equals(testName)) { + return "referencechains=true:hops=32:budget=500:framecap=1:ttl=100:painbudget=100"; + } + // Deliberately does not request cpu/wall/memory/nativemem: those categories also + // write an "enabled" ActiveSetting (flightRecorder.cpp:1141-1144) and all default to + // false when not requested, so "enabled"="true" is unambiguous evidence of the + // datadog.ReferenceChain setting specifically without needing to disambiguate by the + // ActiveSetting "id" field (JMC's generic accessor lookup does not resolve that field + // for this custom event type - not worth a bespoke accessor for one assertion). + return "referencechains=true:hops=32:budget=500:ttl=2000:framecap=256"; + } + + @Override + protected boolean isPlatformSupported() { + // FollowReferences/tag-based frontier walking (referenceChains.cpp) assumes a + // HotSpot-shaped JVMTI heap implementation; excluded platforms mirror + // LivenessTrackingTest's own guard (memleak/LivenessTrackingTest.java). + return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); + } + + /** + * Verifies the {@code referencechains=...} flag round-trips through + * {@code Arguments} parsing (arguments.cpp's {@code CASE("referencechains")}) into the + * {@code datadog.ReferenceChain} JFR setting (flightRecorder.cpp:1143's + * {@code writeBoolSetting(buf, T_REFERENCE_CHAIN, "enabled", args._reference_chains)}). + */ + @RetryingTest(5) + public void shouldExposeReferenceChainsSettingWhenEnabled() { + stopProfiler(); + JfrEvents settings = verifyEvents("jdk.ActiveSetting"); + boolean sawEnabledSetting = false; + for (JfrEvent item : settings) { + if ("enabled".equals(item.getString("name")) && "true".equals(item.getString("value"))) { + sawEnabledSetting = true; + } + } + assertTrue(sawEnabledSetting, "datadog.ReferenceChain#enabled setting was not found"); + } + + /** + * This test's stated success-path scenario, now exercising the Remaining Work Plan's + * target-selection bridging feed instead of staying disabled: allocates a growing population of + * {@link ChainLink} instances, forcing GCs between allocation rounds so + * {@code LivenessTracker::cleanup_table()}'s epoch-advance pass (livenessTracker.cpp) observes a + * rising per-klass survivor count each time, until {@code selectLeakCandidates()} trusts the + * resulting trend (needs {@code KLASS_POPULATION_MIN_FILL_FOR_TREND = 10} ring-buffer samples). + * Then waits for {@code ReferenceChainTracker::pollWatchedTargets()} to notice that ranked + * candidate has already been tagged by an ordinary {@code runPass()} walk and reconstruct + + * emit its chain, and asserts on the resulting {@code datadog.ReferenceChain} event. + * + *

    Each population-growth round pairs with an explicit {@link #dump(Path)}: per + * livenessTracker.cpp, {@code cleanup_table()}'s per-klass population accounting only runs + * (unforced) from {@code flush_table()}, which only runs from {@code LivenessTracker::flush()}, + * which is called *only* from {@code Profiler::dump()} - there is no timer-driven flush. + * {@code System.gc()} bumps {@code LivenessTracker::_gc_epoch} synchronously inside the + * {@code GarbageCollectionFinish} callback ({@code onGC()}), so by the time {@code System.gc()} + * returns to Java the epoch bump is already visible - no extra sleep is needed between the + * {@code gc()} and the {@code dump()} that observes it. + */ + @Test + @Order(1) + public void shouldReconstructReferrerChainToGcRoot() throws Exception { + // Seed gcRootHolder with one live element *before* resetting the search below: ArrayList's + // backing array starts out as the shared empty-array sentinel, and only gets replaced with a + // real array on its first grow() (addAll()). ReferenceChainTracker::expandFrontier() + // (referenceChains.cpp) freezes whatever children it observes for a node the moment that + // node is expanded - it never re-examines an already-EXPANDED node for a later field mutation, + // regardless of how many times root/stack-ref enumeration itself reruns over the search's + // lifetime (runPassManualWalk()'s own comment). If gcRootHolder's node gets expanded while + // elementData still pointed at the empty sentinel (nothing stops some *unrelated* GC in this + // shared, no-forkEvery JVM from waking the freshly (re)started BFS thread in the gap between + // resetting the search below and round 1's own addAll()), the frozen children set would be + // permanently empty, and no ChainLink added in any of the 16 rounds that follow would ever + // become reachable - matching a real CI failure where pollWatchedTargets() reported tag=0 for + // this klass on every single poll across the whole test. Seeding one element first means the + // backing array is never empty at any point after the search (re)starts: every later + // addAll()/grow() copies all prior elements (including this one) into the new array, so + // whichever array the walk happens to snapshot when it expands the node, index 0 is always + // present in it. + gcRootHolder = new ArrayList<>(); + gcRootHolder.add(new ChainLink("gc-root-seed")); + if ("debug".equals(System.getProperty("ddprof_test.config"))) { + // Being pinned to run first *within this class* (this class's own header comment, "Why + // @TestMethodOrder/@Order(1)") does not guarantee this is the first reference-chain test + // to ever call into the shared, process-wide singleton ReferenceChainTracker in this whole + // test JVM - no forkEvery is configured (same header comment), so another test class can + // run first and leave the singleton search already non-RUNNING/already-tagged, in which + // case runPass() never takes its real root-seeded-walk branch again. Force a genuine fresh + // search so this test's own ChainLink population is guaranteed reachable by a real walk, + // regardless of what ran earlier in this JVM. Debug-only: this native seam does not exist + // in a release build. + // + // resetKlassPopulationForTest0() first, for the same "no forkEvery" reason: + // LivenessTracker's _klass_population table (livenessTracker.cpp) also survives stop()/ + // start() cycles, so an unrelated earlier test class in this same shared JVM (e.g. another + // generations=true scenario) could leave a klass already past hasLeakSignal()'s + // consecutive_positive hysteresis gate (selectLeakCandidates(), livenessTracker.cpp) + // sitting in that table - which would let this test's first pass start for a reason that + // has nothing to do with the seeding this method does further down (see + // SEED_EPOCHS_FOR_HYSTERESIS's own comment), masking whether that seeding is actually + // sufficient on its own. + JavaProfiler.resetKlassPopulationForTest0(); + JavaProfiler.resetReferenceChainSearchForTest0(); + } + // One persistent allocator thread, reused by every round - NOT a fresh + // spawned thread per round. Same JVMTI property the per-round spawn + // relied on (SampledObjectAlloc never fires for this JUnit worker + // thread's own allocations, but reliably fires for threads created + // after Profiler::start() already ran), plus the decisive per-tid + // consequence: LivenessTracker's per-(klass, tid) qualification trend + // accumulates REAL samples for this one thread's tid every round + // (foldKlassCountsLocked() folds each tracked instance's tid), so + // qualification is organic and cannot age out mid-test. The previous + // per-round spawns gave every round a DIFFERENT short-lived tid, so + // the seeded (klass, worker-tid) qualification ramp was the only + // qualifying signal - and once its ring slots aged out (observed + // live: slope decayed to ~0 at ring_fill=22, consecutive_positive + // reset, the candidate never returned again) the candidate dropped + // out of selectLeakCandidates() with the whole crawl still unfinished. + LinkedBlockingQueue allocRequests = new LinkedBlockingQueue<>(); + Semaphore roundDone = new Semaphore(0); + AtomicLong allocatorTid = new AtomicLong(-1); + Thread allocator = new Thread(() -> { + allocatorTid.set(JavaProfiler.getTid()); + try { + Integer requested; + while ((requested = allocRequests.take()) != null) { + List batch = currentBatch; + for (int i = 0; i < requested; i++) { + batch.add(new ChainLink("leak-" + requested + "-" + i)); + } + roundDone.release(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // test shutdown - see finally below + } + }, "togcroot-leak-allocator"); + allocator.start(); + // The seeding block below needs the allocator's tid; the thread publishes + // its own tid as its first action, so this wait is bounded by thread start. + while (allocatorTid.get() == -1) { + Thread.sleep(1); + } + + Path scratchDumpPath = Paths.get("referencechains-population-scratch.jfr"); + try { + // Up to 16 rounds: selectLeakCandidates()'s KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 + // (livenessTracker.h) needs 10 *epochs that actually observe a surviving ChainLink sample*, + // not just 10 dump() calls - allocation sampling is probabilistic (see below), so some + // early, smaller rounds may not land a single ChainLink sample. Growing the per-round count + // compensates, and this loop keeps going (checking for the event every round) rather than + // committing to a fixed round count up front, so it self-adjusts to whatever this JVM's + // actual sampling behavior turns out to be. Staying under KLASS_POPULATION_RING_SIZE = 30 + // keeps every round's sample within the trend computeKlassPopulationSlope() reads back out. + // Capped at 16 rather than a larger margin above the 10-round minimum: this loop's own + // worst case (every round retained, no early match) runs inside the same forked test JVM + // every other ddprof-test class shares (ProfilerTestPlugin.kt's shared -Xmx512m default, + // no forkEvery) - a prior CI run hit "Java heap space" in that shared fork with this loop + // capped at 25, so the cap trades some of the original margin above the 10-round minimum + // for staying inside that shared heap. + // + // Per-round size growth itself is clamped to round 10 (Math.min(round, 10) below): rounds + // past 10 exist only to give pollWatchedTargets() more retries against the lock-contention + // race described below, not to keep building the population trend (already eligible by + // round 10) - letting round*600 keep scaling unclamped through round 16 made rounds 11-16 + // each add strictly more retained garbage than the last for no trend benefit, which is what + // drove the "Java heap space" failure this comment's own history refers to. + // + // Instance count is sized against ObjectSampler::check() (objectSampler.cpp), not this + // test's own "memory=64" request: "do not allow shorter interval than 256KiB" means the + // *actual* allocation-sampling interval is 262144 bytes regardless of the small value + // requested above (that request only affects LivenessTracker's own table-capacity formula, + // livenessTracker.cpp's initialize_table()). ChainLink's own padding fields (see that + // class's comment) get enough total megabytes sampled from far fewer instances than plain, + // unpadded ~40-byte instances would need - keeping this test's own contribution to the real + // JVM's root-reachable graph (which ReferenceChainTracker's BFS walk must also traverse, + // Triggering section) from ballooning to the point the walk can't practically catch up + // within this test's own wall-clock budget. ChainLink competes on equal footing against + // other incidental klasses (e.g. "[B"/byte[]) that this same mechanism may also legitimately + // flag as leak candidates - this test's own assertions below look for ChainLink specifically + // among however many datadog.ReferenceChain events actually appear, rather than assuming it + // is the only one. + ReferenceChainAssertions.JfrChainMatch match = null; + boolean seededTestKlassTrend = false; + int totalRounds = 16; + for (int round = 1; round <= totalRounds && match == null; round++) { + int newInstances = Math.min(round, 10) * 600; + List newLinks = new ArrayList<>(newInstances); + currentBatch = newLinks; // published before the request - see the allocator's loop + if (!allocRequests.offer(newInstances)) { + throw new IllegalStateException( + "allocRequests queue rejected instance count " + newInstances); + } + roundDone.acquireUninterruptibly(); + gcRootHolder.addAll(newLinks); + System.gc(); + dump(scratchDumpPath); + match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); + + if (match == null && "debug".equals(System.getProperty("ddprof_test.config"))) { + // Debug-only native seams - a no-op in release builds, so this leaves release-build + // coverage of the organic path below completely unchanged. + // + // Seeding a klass_id's population ring here is not just a ranking shortcut for an + // object some other pass has already tagged - with generations=true, + // ReferenceChainTracker::shouldRunPass() (referenceChains.cpp) gates even this search's + // very first pass on LivenessTracker::hasLeakSignal(), which (once secondsToOOM() has + // nothing to report yet, this early) falls through to selectLeakCandidates() requiring + // consecutive_positive >= LEAK_TREND_HYSTERESIS_BASE for *some* klass, real or not (see + // SEED_EPOCHS_FOR_HYSTERESIS's own comment for why 10 seeded epochs alone cannot clear + // that bar). Seeding SEED_EPOCHS_FOR_HYSTERESIS epochs for this test-chosen id clears it + // outright, which is what actually authorizes the background BFS thread's next + // scheduled wake to run its real, root-seeded runPass() walk - nothing before that point + // has tagged gcRootHolder's own ChainLink yet. pollReferenceChainTargets0() right below + // is therefore best-effort, not guaranteed on this call: it only reconstructs a chain + // once getTag() on the representative is actually > 0 (pollWatchedTargets()'s only + // precondition), which needs that background pass to have already run. The retry loop + // below (a 300ms-sleep-then-dump fallback, repeated across rounds) gives the + // now-unblocked background thread room to do that and have its own post-pass + // pollWatchedTargets() call (threadLoop(), referenceChains.cpp) pick the chain up on its + // own, independent of this method's own poll call succeeding on the first try. + // + // Seed exactly once (recordKlassPopulationSampleLocked(), livenessTracker.cpp, always + // *appends* a fresh ring slot rather than overwriting one for a repeated epoch - calling + // this whole block again on a later round would append a second ramp right after the + // first, turning the ring into a non-monotonic sawtooth and destroying the very + // positive-slope signal selectLeakCandidates() needs). Only the poll+dump recheck below + // needs to repeat across rounds - not the seeding - to give the background pass, or a + // lock-contended dump(), more rounds to resolve. + // Representative BEFORE seeding: setKlassPopulationRepresentativeForTest0 + // resolves the representative's real klass id and aliases the synthetic + // id to it, so the hysteresis seeds below land in the real population + // entry (candidate matching is real-id keyed since the leak-tag pool + // redesign; seeds under a bare synthetic id authorize only + // hasLeakSignal()'s candidate-count check and match nothing else). + JavaProfiler.setKlassPopulationRepresentativeForTest0(CHAIN_LINK_TEST_KLASS_ID, gcRootHolder.get(0)); + if (!seededTestKlassTrend) { + // Per-(klass, tid) qualification seeds - the persistent allocator + // thread owns every ChainLink allocation, so ITS tid is the + // qualifying one; the seeds accelerate the organic per-tid growth + // its own tracked instances produce every round (see the + // allocator block's own comment for why qualification aging out + // stranded the crawl before this thread existed). + int leakTid = (int) allocatorTid.get(); + // Seed counts scale as the organic ramp itself (+1 per epoch), + // NOT some inflated multiple: the ring is one continuous series + // of seeds followed by the real folds, and hasQualifyingGrowth() + // takes the difference of third-window MEANS over that whole + // series. Seeds of epoch*10 (10..150) dwarf the real generation + // counts (the real ramp is +1 per epoch, ~1..15 by test end), so + // the moment real samples displace the seeded tail the slope + // went NEGATIVE (-20..-40 observed live: the "transition cliff") + // and the candidate never returned again. With matched-scale + // seeds the series is [1..15, 1, 2, 3, ...] and every third- + // window pairing across the transition still yields a positive + // slope - the candidate keeps qualifying organically the whole + // run, the real signal seamlessly taking over from the seeds. + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(CHAIN_LINK_TEST_KLASS_ID, epoch, epoch); + JavaProfiler.seedTidTrendSample0(CHAIN_LINK_TEST_KLASS_ID, leakTid, epoch, epoch); + } + seededTestKlassTrend = true; + } + JavaProfiler.pollReferenceChainTargets0(); + dump(scratchDumpPath); + match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); + } + + if (match == null) { + // A quiet window with no dump() in flight: Profiler::dump() (profiler.cpp) holds every + // entry of _locks[] exclusively for the duration of its own _jfr.dump() call + // (rotateDictsAndRun()'s lockAll()/unlockAll() pair) - the same array + // Profiler::writeReferenceChain() (profiler.cpp) needs a slot from to record an event at + // all. Back-to-back rounds with essentially no gap between one dump() and the next leave + // pollWatchedTargets() (referenceChains.cpp) little to no window to ever win that race; + // sleeping briefly here, then dumping again with no intervening dump()/gc() in between + // (so cleanup_table()'s epoch-advance pass, livenessTracker.cpp, stays a no-op and the + // population trend built up so far is undisturbed), gives it one. + Thread.sleep(300); + dump(scratchDumpPath); + match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); + } + } + + // Grace period: the population trend only became eligible (ring_fill >= 10) partway through + // the loop above, and the same lock contention the loop's own comment describes can still + // delay the resulting write past this method's very last dump() - retry a few more times, + // well past that contention window, before concluding the mechanism genuinely did not fire. + for (int attempt = 0; match == null && attempt < 5; attempt++) { + Thread.sleep(1000); + dump(scratchDumpPath); + match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); + } + + assertNotNull(match, + "Never observed a datadog.ReferenceChain event whose chain[0] is " + ChainLink.class + + " after " + gcRootHolder.size() + " retained ChainLink instances across up to " + + totalRounds + " population-growth rounds plus a grace period"); + // chain[0] is the *target* object's own class, not its holder's: + // ReferenceChainTracker::buildChainEvent()'s reconstructChain() (referenceChains.h) + // appends each visited FrontierEntry's referrer_klass leaf-to-root, and referrer_klass is + // populated from the *discovered object's own* GetClassSignature at insertion time (see + // referenceChainJfrRoundtrip_ut.cpp's insert(tag, parent_tag, referrer_klass, depth) calls, + // where tag 3's own referrer_klass is leafKlass, not middleKlass). The target here is + // always some currently-live ChainLink instance - the only class this test's leak-candidate + // population trend was built from - regardless of exactly which of the many instances + // allocated above LivenessTracker::foldKlassCountsLocked() (livenessTracker.cpp) happened to + // keep as its representative. Everything above chain[0] reflects real JDK-internal + // collection representation (e.g. ArrayList's backing array) rather than anything this test + // controls, so it is deliberately not asserted beyond "at least one hop was reconstructed". + assertEquals(ChainLink.class.getName(), match.chain.get(0)); + assertTrue(match.targetTag > 0, "targetTag should be a valid, non-zero JVMTI tag"); + assertTrue(match.depth >= 0, "depth should be a non-negative hop count"); + assertTrue(!gcRootHolder.isEmpty()); // keeps every allocated ChainLink reachable until here + } finally { + gcRootHolder = null; // shared-JVM hygiene - see the static holder's own comment + currentBatch = null; + // Shut the persistent allocator thread down before leaving: this is a + // shared, no-forkEvery test JVM, and a leftover allocation thread would + // keep allocating (and keep its tid trend alive) into every later test. + allocator.interrupt(); + allocator.join(1000); + Files.deleteIfExists(scratchDumpPath); + } + } + + /** + * A second, independent end-to-end scenario for the same target-selection mechanism as + * {@link #shouldReconstructReferrerChainToGcRoot()}, using a shape much closer to a real-world + * leak than that method's hand-rolled {@link ChainLink} list: an ever-growing, never-evicted + * {@link java.util.HashMap}-backed cache - a very common real leak pattern - holding + * {@link CachedPayload} values. Beyond confirming a chain fires for the leaking value type, + * this also asserts the reconstructed chain actually threads back through + * {@code java.util.HashMap}'s own internal storage ({@code HashMap$Node}), i.e. that the + * mechanism correctly walks a real JDK collection's internals rather than only ever having + * been proven against a single purpose-built linked fixture. + * + *

    Why {@code cache} is a local variable, not a {@code static} field (found the hard + * way): {@code ReferenceChainTracker::expandFrontier()} (referenceChains.cpp) freezes + * whatever children it observes for a node the moment that node is expanded - it never + * re-examines an already-{@code EXPANDED} entry for newly-added children, regardless of how + * many times root/stack-ref enumeration itself reruns over the search's lifetime + * ({@code runPassManualWalk()}'s own comment). A {@code static} field becomes a root at + * class-load time, essentially guaranteeing some early pass catches it (and marks it + * {@code EXPANDED}) while still empty - permanently blocking discovery of anything added to it + * afterward, no matter how many rounds run. A local variable created fresh at the top of this + * method, immediately followed by round-1 allocation, gives root enumeration a real chance to + * catch it only after it already holds live data - mirroring + * {@link #shouldReconstructReferrerChainToGcRoot()}'s {@code gcRootHolder}, which works for + * exactly this reason (a local {@code ArrayList}, not a {@code static} one). + * + *

    See this class's own header comment for why this runs as {@code @Order(2)}, before + * {@link #shouldReportAbandonedSearchOnTinyFrontierCap()}, and for why its relative order + * against {@link #shouldReconstructReferrerChainToGcRoot()} does not itself matter. + */ + @Test + @Order(2) + public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exception { + // Seed cache with one live entry *before* the resets below, mirroring + // shouldReconstructReferrerChainToGcRoot()'s own gcRootHolder seeding (see that method's + // comment): HashMap's backing table also starts out as a shared, lazily-replaced empty + // sentinel (only allocated on the first put()), so the same one-shot-BFS-freeze hazard + // documented there - a concurrent GC-triggered walk expanding this node's children while the + // backing table is still the empty sentinel, permanently freezing an empty children set - + // applies here too. Seeding one entry first means the backing table is never empty at any + // point after the search (re)starts. + Map cache = new HashMap<>(); + cache.put("cache-leak-seed", new CachedPayload("cache-leak-seed")); + if ("debug".equals(System.getProperty("ddprof_test.config"))) { + // Mirrors shouldReconstructReferrerChainToGcRoot()'s own reset (see that method's comment): + // the natural restartSearch() cycle (shouldRunPass(), referenceChains.cpp) that would + // otherwise give this method its own fresh root walk once shouldReconstructReferrerChainToGcRoot()'s + // own ChainLink candidate is found and its tags released is gated on cadence/pacing budget + // (canAffordNewSearch()) - not guaranteed to fire again before this method's own round/retry + // budget runs out. Force a genuine fresh search here too, rather than depend on that timing, + // so cache (below) is guaranteed reachable by a real walk regardless of it. Debug-only: this + // native seam does not exist in a release build. + // + // resetReferenceChainSearchForTest0() only resets ReferenceChainTracker's own search state + // (frontier/tags/search-progress fields); it does not touch LivenessTracker's per-klass + // population-history rings, which selectLeakCandidates() consults independently to decide + // which klass is "trending". Without also resetting those, a klass ChainLink already + // accumulated a positive slope for during shouldReconstructReferrerChainToGcRoot() can still + // outrank CachedPayload as the leak candidate here, so the fresh search below ends up + // reconstructing ChainLink's chain again instead of CachedPayload's. + JavaProfiler.resetKlassPopulationForTest0(); + JavaProfiler.resetReferenceChainSearchForTest0(); + } + Path scratchDumpPath = Paths.get("referencechains-cache-leak-scratch.jfr"); + try { + // Same round-growth/retry shape as shouldReconstructReferrerChainToGcRoot() - see that + // method's own comment for why the loop self-adjusts rather than committing to a fixed + // round count, why per-round scale is sized against ObjectSampler's real 256KiB sampling + // floor rather than this test's own "memory=64" request, why totalRounds is capped at + // 16 rather than a larger margin above the 10-round minimum (shared-fork heap headroom), + // and why per-round growth itself is clamped to round 10 (Math.min(round, 10) below). + ReferenceChainAssertions.JfrChainMatch match = null; + boolean seededTestKlassTrend = false; + int totalRounds = 16; + + // Pre-generate every key this loop will ever need, up front, rather than concatenating a + // fresh String on every put() below. CachedPayload's own class comment already documents + // why this fixture uses plain long fields instead of a byte[] (an earlier version's byte[] + // field stole every allocation-sampling hit from CachedPayload itself) - the key String + // (plus its own backing byte[]) is the same kind of same-instant, size-weighted-sampler + // competitor, just a *separate* object instead of a field. Generating all keys in one + // batch before the round loop starts lets their own klass_population trend flatten out + // (selectLeakCandidates() needs KLASS_POPULATION_MIN_FILL_FOR_TREND=10 *new* samples, not + // just a nonzero ring) well before CachedPayload's own ring starts growing, so the sampler + // has nothing else in flight to compete with once round 1 begins. + int maxEntries = 0; + for (int round = 1; round <= totalRounds; round++) { + maxEntries += Math.min(round, 10) * 600; + } + String[] keys = new String[maxEntries]; + for (int i = 0; i < keys.length; i++) { + keys[i] = "leak-" + i; + } + + int nextKey = 0; + for (int round = 1; round <= totalRounds && match == null; round++) { + // Matches shouldReconstructReferrerChainToGcRoot()'s own *600 growth rate (not *300, this + // method's previous value) - CachedPayload and ChainLink are near-identical in size (same + // padding fields; CachedPayload is actually slightly smaller, missing ChainLink's `next` + // reference), so halving the growth rate here bought no headroom and instead just let + // CachedPayload's own population ring stall short of KLASS_POPULATION_MIN_FILL_FOR_TREND + // within the same totalRounds budget both methods share. + int newEntries = Math.min(round, 10) * 600; + int keyOffset = nextKey; + Thread allocator = new Thread(() -> { + for (int i = 0; i < newEntries; i++) { + String key = keys[keyOffset + i]; + cache.put(key, new CachedPayload(key)); + } + }); + allocator.start(); + allocator.join(); + nextKey += newEntries; + System.gc(); + dump(scratchDumpPath); + JfrEvents events1 = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); + match = ReferenceChainAssertions.findMatchForClass(events1, CachedPayload.class); + + if (match == null && "debug".equals(System.getProperty("ddprof_test.config"))) { + // Same deterministic short-circuit as shouldReconstructReferrerChainToGcRoot() - see + // that method's own comment for the full mechanism (why seeding + // SEED_EPOCHS_FOR_HYSTERESIS epochs is what actually authorizes the background BFS + // thread's next real pass, and why this method's own pollReferenceChainTargets0() call + // right below is therefore best-effort rather than something the seed alone + // guarantees on this call). Seed exactly once - see that method's own comment for why + // reseeding the same ramp on a later round would corrupt the ring into a non-monotonic + // sawtooth and destroy the positive-slope signal instead of just re-establishing it. + // Representative BEFORE seeding - same real-id aliasing rationale as + // shouldReconstructReferrerChainToGcRoot()'s own comment above. + JavaProfiler.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get(keys[0])); + if (!seededTestKlassTrend) { + // Per-(klass, tid) qualification seeds - see LeakingCacheScenario's + // own seeding block for why this thread's tid (the representative's + // creator) is the qualifying one for the one-cohort-per-round shape. + int leakTid = JavaProfiler.getTid(); + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedTidTrendSample0(CACHED_PAYLOAD_TEST_KLASS_ID, leakTid, epoch * 3, epoch); + } + seededTestKlassTrend = true; + } + JavaProfiler.pollReferenceChainTargets0(); + dump(scratchDumpPath); + JfrEvents events2 = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); + match = ReferenceChainAssertions.findMatchForClass(events2, CachedPayload.class); + } + + if (match == null) { + // Same lock-contention race shouldReconstructReferrerChainToGcRoot()'s own comment + // describes - a quiet retry gives pollWatchedTargets() a window to win a _locks[] + // slot against Profiler::dump()'s own exclusive hold. + Thread.sleep(300); + dump(scratchDumpPath); + JfrEvents events3 = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); + match = ReferenceChainAssertions.findMatchForClass(events3, CachedPayload.class); + } + } + + for (int attempt = 0; match == null && attempt < 5; attempt++) { + Thread.sleep(1000); + dump(scratchDumpPath); + JfrEvents events4 = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); + match = ReferenceChainAssertions.findMatchForClass(events4, CachedPayload.class); + } + + assertNotNull(match, + "Never observed a datadog.ReferenceChain event whose chain[0] is " + CachedPayload.class + + " after " + cache.size() + " cached entries across up to " + totalRounds + + " population-growth rounds plus a grace period"); + assertEquals(CachedPayload.class.getName(), match.chain.get(0)); + assertTrue(match.targetTag > 0, "targetTag should be a valid, non-zero JVMTI tag"); + assertTrue(match.depth >= 0, "depth should be a non-negative hop count"); + + // The point of this test over shouldReconstructReferrerChainToGcRoot(): confirm the walk + // actually passed through the cache's own internal storage, not some other, coincidental + // retainer - cache is the only thing keeping any CachedPayload instance reachable. + boolean sawHashMapInternals = false; + for (String type : match.chain) { + if (type.startsWith("java.util.HashMap")) { + sawHashMapInternals = true; + break; + } + } + assertTrue(sawHashMapInternals, + "Expected the reconstructed chain to pass through java.util.HashMap's own internal " + + "storage (cache is a HashMap) - chain was: " + match.chain); + assertTrue(!cache.isEmpty()); // keeps every cached CachedPayload reachable until here + } finally { + Files.deleteIfExists(scratchDumpPath); + } + } + + /** + * This test's stated abandonment-path scenario: an artificially tiny frontier cap + * ({@code getProfilerCommand()}'s {@code framecap=1} for this test method, see that method's own + * comment for why {@code budget} must stay larger than {@code framecap}) makes the very first + * BFS pass hit the frontier cap once a second root-referenced object is discovered, so + * {@code runPass()} (referenceChains.cpp) abandons the search rather than silently truncating it + * - or, if this method's own + * {@code framecap=1} lost the race to size the shared {@code FrontierTable} (see + * {@code getProfilerCommand()}'s own comment on this method), its {@code ttl=100} fallback + * abandons the search instead once enough wall-clock time has passed, which by the time this + * method runs it always has. Either path exercises the same {@code buildAbandonedEvent()} / + * {@code Profiler::writeReferenceChainAbandoned()} / {@code FlightRecorder::recordReferenceChainAbandoned()} + * write into the recording that {@code Profiler::dump()} triggers - this method only asserts that + * a {@code datadog.ReferenceChainAbandoned} event exists, not which reason produced it. + * + *

    {@code @Order(3)}: this permanently exhausts the process-wide {@code ReferenceChainTracker} + * singleton's one-and-only search (see this class's header comment), so it must run after + * {@link #shouldReconstructReferrerChainToGcRoot()} and + * {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}. It used to rely on being + * the last method in source order plus JUnit's default (unannotated methods sort after + * {@code @Order}-annotated ones) for that - which happened to guarantee run order, but not the + * search *state* this method's own comment above presumes: if either earlier test's own + * organic GC/allocation-sampling trend never fired in time (observed in practice - real + * flakiness, not this test's fault), the shared search can still be {@code RUNNING} with a + * large, non-fresh frontier when this method starts, instead of the small, just-abandoned one + * its own {@code framecap=1}/{@code ttl=100} scenario assumes. Calling + * {@code resetReferenceChainSearchForTest0()} here, exactly as both earlier tests already do for + * their own scenarios, makes this method's own precondition (a fresh search) something it + * establishes itself rather than something it presumes a prior test left behind. Debug-only: + * this native seam does not exist in a release build. + * + *

    Why this method also seeds a klass population trend, despite requesting no + * {@code generations=true} of its own (found the hard way: {@code resetReferenceChainSearchForTest0()} + * alone was not enough - the search's first pass never ran at all, so it could never reach + * {@code SearchState::ABANDONED} for {@code dump()} to observe): {@code LivenessTracker::_gc_generations} + * (livenessTracker.cpp's {@code initialize()}) is only ever updated when a {@code start()} command + * actually requests liveness tracking - a bare {@code referencechains=true:...} command like this + * method's own does not call {@code LivenessTracker::start()} at all, so + * {@code gcGenerationsEnabled()} stays stuck at whatever the two {@code generations=true} tests + * above last left it (confirmed directly: this method's own native trace shows + * {@code gc_generations=1}). {@code ReferenceChainTracker::hasLeakSignal()}, which + * {@code shouldRunPass()} consults for even this search's very first pass, therefore falls + * through to {@code selectLeakCandidates()} exactly as the two tests above do - with nothing + * left over from {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}'s own + * {@code resetKlassPopulationForTest0()} call to satisfy it. Seeding + * {@code SEED_EPOCHS_FOR_HYSTERESIS} epochs for a dedicated klass id (see + * {@code SEED_EPOCHS_FOR_HYSTERESIS}'s own comment for the full derivation) clears that gate the + * same way it does there - this method never sets a representative or polls for it, since it has + * no reconstructed chain of its own to prove; {@code selectLeakCandidates()} + * (livenessTracker.cpp) counts a qualifying entry toward {@code hasLeakSignal()}'s candidate + * count regardless of whether its representative was ever set, and + * {@code resolveCandidateRepresentative()} is null-safe against one that wasn't. + * + *

    Why the seeding is preceded by a {@code System.gc()} and a throwaway {@code dump()} + * (found the hard way, twice: a bare {@code dump()} with no preceding GC still lost the + * race): {@code cleanup_table()} (livenessTracker.cpp) compares + * {@code Profiler::classMap()->generation()} against its own cached copy and, on a mismatch - + * which this method's own fresh {@code start()} causes, by clearing that map - wipes + * {@code _klass_population} back to empty before folding anything else, exactly once per + * mismatch; but that comparison sits *after* an early-return guard + * (`{@code if (!is_epoch_owner && !forced) return;}`) that a {@code dump()}-triggered call + * (always {@code forced=false}) only clears when a genuinely new, not-yet-processed GC epoch + * is pending - a {@code dump()} with no GC since the last {@code cleanup_table()} pass never + * reaches the mismatch check at all. The two methods above never hit any of this because their + * own seed loop runs only after their round-1 {@code System.gc()}+{@code dump()} pair has + * already triggered that one-time wipe-and-resync as a side effect of their own allocation + * loop; seeding before any GC/dump of this method's own would instead leave the seed to be + * discarded by the *next* call that does clear the guard - this class's own + * {@code maybeForceCleanup()} tick (called once per {@code ReferenceChainTracker::threadLoop()} + * wake, {@code forced=true} unconditionally clears the guard), which fires on this method's own + * 3 {@code System.gc()} calls below regardless. The explicit {@code System.gc()}+{@code dump()} + * pair here forces that one-time wipe-and-resync to happen now, before seeding, exactly like + * the two methods above get for free from their own round-1 allocation loop. + */ + @Test + @Order(3) + public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { + if ("debug".equals(System.getProperty("ddprof_test.config"))) { + // See this method's own javadoc ("Why this method also seeds a klass population trend...") + // for why this is needed even though this method requests no generations=true of its own. + // + // A System.gc() + throwaway dump() first - see this method's own javadoc ("Why the seeding + // is preceded by...") for why *both* are needed: a dump() alone triggers cleanup_table() + // with forced=false, which only clears the class-map-generation-mismatch guard (and + // therefore only wipes/resyncs _klass_population) when a new GC epoch is actually pending - + // this System.gc() is what makes that true for this call, rather than leaving it to chance + // (or to this method's own 3 System.gc() calls below, which run after the seed and would + // let maybeForceCleanup()'s forced=true tick discard it first). + System.gc(); + Path warmupDumpPath = Paths.get("referencechains-abandoned-test-warmup.jfr"); + try { + dump(warmupDumpPath); + } finally { + Files.deleteIfExists(warmupDumpPath); + } + JavaProfiler.resetKlassPopulationForTest0(); + // Per-(klass, tid) qualification seeds - gate-only usage (this test + // drives the search to ABANDONED and asserts on the abandon marker, no + // tagged instance needs to match), so the tid is this thread's real one + // but any distinct tid would do. + int abandonTid = JavaProfiler.getTid(); + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(ABANDON_TEST_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedTidTrendSample0(ABANDON_TEST_KLASS_ID, abandonTid, epoch * 3, epoch); + } + JavaProfiler.resetReferenceChainSearchForTest0(); + } + List gcRootHolder = new ArrayList<>(); + gcRootHolder.add(new ChainLink("middle", new ChainLink("leaf"))); + + // GarbageCollectionFinish (onGCFinish(), referenceChains.cpp) wakes the BFS thread + // early, but that wakeup can race the thread's own startup (VM::attachThread() + // completing before its first OS::sleep() call) and be missed. Don't rely on the + // signal alone: sleep comfortably past ReferenceChainTracker::PASS_CADENCE_NS (1s, + // referenceChains.h) too, so the thread's fixed-cadence fallback trigger + // (shouldRunPass()) guarantees at least one pass runs regardless of that race. + for (int i = 0; i < 3; i++) { + System.gc(); + Thread.sleep(100); + } + Thread.sleep(1500); + + Path dumpPath = Paths.get("referencechains-abandoned-test.jfr"); + try { + // Even with the seeding above clearing hasLeakSignal()'s gate so the search's first pass is + // actually allowed to start (this method's own javadoc), that pass still has to be woken by + // the BFS thread and still has to actually reach the frontier cap/ttl cutoff, inside whatever + // real wall-clock time this shared, no-forkEvery test JVM's own scheduling happens to grant + // it (this class's own header comment) - a single fixed-length sleep-then-dump-then-assert, + // with no room to retry, previously failed outright whenever that one attempt landed a beat + // early. Retry across a few more dumps, spaced far enough apart to tolerate a slow + // BFS-thread wake or a walk over a larger-than-usual inherited heap, before concluding the + // search genuinely never abandoned - mirroring the bounded retry loops + // shouldReconstructReferrerChainToGcRoot()/shouldReconstructReferrerChainThroughUnboundedCacheLeak() + // already use above. + JfrEvents abandoned = null; + for (int attempt = 0; attempt < 8; attempt++) { + dump(dumpPath); + abandoned = verifyEvents(dumpPath, "datadog.ReferenceChainAbandoned", false); + if (abandoned.hasItems()) { + break; + } + Thread.sleep(1000); + } + assertTrue(abandoned != null && abandoned.hasItems(), + "Expected at least one datadog.ReferenceChainAbandoned event after 3 GCs, an initial " + + "1500ms grace period, and 8 retries 1s apart"); + } finally { + Files.deleteIfExists(dumpPath); + } + assertTrue(!gcRootHolder.isEmpty()); // keeps gcRootHolder reachable until the dump above + } + + /** + * Referrer-type fixture shared by both the success-path and abandonment-path tests. The 32 + * {@code long} fields below exist purely so {@link #shouldReconstructReferrerChainToGcRoot()} + * needs far fewer instances to allocate a given number of megabytes of ChainLink - deliberately + * plain fields, not a nested array: an array field would be a *second*, separate heap + * allocation, and the allocation-sampling interval that method's comment describes picks + * whichever allocation happens to cross its byte threshold size-weighted, so a same-instance + * companion array would take sampling attention away from ChainLink itself rather than adding + * to it. Plays no role in {@link #shouldReportAbandonedSearchOnTinyFrontierCap()}'s tiny, + * two-object fixture beyond trivially increasing its size. + */ + private static final class ChainLink { + final String name; + final Object next; + long p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15; + long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31; + + ChainLink(String name) { + this(name, null); + } + + ChainLink(String name, Object next) { + this.name = name; + this.next = next; + } + } + + /** + * Realistic-leak fixture for {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}: + * models the single most common real-world leak shape this mechanism is meant to catch - an + * unbounded, never-evicted cache - as opposed to {@link ChainLink}'s hand-rolled linked list. + * The 32 {@code long} fields exist purely so a given number of megabytes needs far fewer + * entries to reach {@code ObjectSampler}'s real 256KiB sampling floor - deliberately plain + * fields, not a nested array, for exactly the reason {@link ChainLink}'s own comment already + * documents: a same-instance companion array would be a *second*, separate heap allocation + * that the size-weighted allocation sampler would compete for, taking sampling attention away + * from {@code CachedPayload} itself (an earlier version of this fixture used a {@code byte[]} + * field and never observed a single {@code CachedPayload} sample as a result - the sampler was + * catching the array instead). + */ + private static final class CachedPayload { + final String key; + long p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15; + long p16, p17, p18, p19, p20, p21, p22, p23, p24, p25, p26, p27, p28, p29, p30, p31; + + CachedPayload(String key) { + this.key = key; + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java new file mode 100644 index 0000000000..a1251f5fed --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java @@ -0,0 +1,278 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.JavaProfiler; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Mimics the real leak shape found in the {@code prof-analyzer-hotdog-jb} pod's leak generator + * ({@code ProfileAnalyzer}'s {@code LEAK_BUFFER}, profiling-backend repo, branch-local diff): a + * {@code static final List} field whose *instance* is admitted exactly once by + * {@code admitStaticFieldRoots()} (referenceChains.cpp) when the owning class first loads, and + * which is then only ever mutated in place (elements appended) - never reassigned - for the rest + * of the process's life. This is the "new elements added to an already-discovered collection" + * shape, as opposed to {@code LeakingCacheScenario}'s local-variable cache (a stack-local GC + * root, not a static field) - see this scenario's own use to confirm whether + * {@code collectStaleExpandedEntriesForRotation()}'s rotation mechanism actually re-expands + * {@link #LEAK_BUFFER} to pick up elements added after its one-time static-field sweep. + * + *

    Same seed-before-start rationale as {@code LeakingCacheScenario}'s own comment: {@link + * #LEAK_BUFFER} is seeded with a few chunks before the profiler (and therefore {@code + * ReferenceChainTracker}'s BFS thread) is even started, so the process's one-shot root-seeded + * walk can never catch it empty. + */ +public final class StaticFieldGrowingCollectionScenario { + private StaticFieldGrowingCollectionScenario() {} + + /** + * Mirrors production's {@code ProfileAnalyzer.LEAK_BUFFER}: a static field holding a + * {@code synchronizedList}, appended to (never reassigned) for the process's whole + * life. + */ + static final List LEAK_BUFFER = Collections.synchronizedList(new ArrayList<>()); + + // Arbitrary, scenario-chosen klass id - see LeakingCacheScenario.CACHED_PAYLOAD_TEST_KLASS_ID's + // own comment; distinct from that scenario's id since both may run in the same suite (each in + // its own separate child JVM/LivenessTracker population table, but kept distinct regardless). + private static final int LEAK_BUFFER_TEST_KLASS_ID = 987302; + + // See LeakingCacheScenario.SEED_EPOCHS_FOR_HYSTERESIS's own comment for the full derivation: + // LivenessTracker's hysteresis gate needs this many qualifying epochs before + // selectLeakCandidates() reports a candidate at all. + private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; + + // Clears ObjectSampler's real sampling floor on its own - unlike LeakingCacheScenario's + // CachedPayload (long fields only, no byte[]), this scenario's leaked object *is* the byte[] + // itself, so there is no second, competing companion allocation. + private static final int CHUNK_BYTES = 300_000; + + private static final int SEED_CHUNKS = 10; + + /** Printed to stdout, followed by the matched leaf class's name, on success. */ + public static final String FOUND_MARKER = "[chain-found] "; + + /** Printed to stdout (with no class name suffix) if no match was ever observed. */ + public static final String NOT_FOUND_MARKER = "[chain-not-found]"; + + /** + * Seeds {@link #LEAK_BUFFER}, starts the profiler, then keeps appending new {@code byte[]} + * chunks to the same list instance - the exact shape implicated in the live hotdog-pod stall - + * watching specifically for a chain to a chunk appended *after* the static field's one-time + * sweep, until a {@code datadog.ReferenceChain} event for it appears or {@code totalRounds} is + * exhausted. + */ + public static void run(JavaProfiler profiler, String startCommand, Path scratchDumpPath) throws Exception { + for (int i = 0; i < SEED_CHUNKS; i++) { + LEAK_BUFFER.add(new byte[CHUNK_BYTES]); + } + if (startCommand != null && !startCommand.isEmpty()) { + profiler.execute(startCommand); + } + + boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + int hysteresisEpoch = 0; + // Declared at method scope: the per-round maintenance seeding below (in + // its own debugBuild block) reuses this tid. + int leakTid = 0; + // Snapshotted BEFORE seeding anything, while the BFS thread's search has not run a single + // pass yet (referenceChainPassesRunForTest0() reads 0 here) - see the wait loop below for why + // this exact value, not just "nonzero", is what the wait needs to advance past. + int initialPasses = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; + + if (debugBuild) { + // Seed enough hysteresis-qualifying epochs to unlock ReferenceChainTracker::hasLeakSignal()/ + // shouldRunPass() - WITHOUT wiring in a representative object yet (setKlassPopulationRepresentativeForTest0 + // is not called here). hasLeakSignal() only asks selectLeakCandidates() for a nonzero count - + // it never reads/dereferences the representative field - so this step's only job is + // authorizing the BFS thread's first pass to actually start. Doing this BEFORE waiting on + // passesRun() advancing below is load-bearing, found the hard way: a pass only ever runs + // once shouldRunPass() sees a leak signal - waiting on passesRun() before seeding anything is + // a deadlock, not just a slow path. + // Per-(klass, tid) qualification seeds (see LeakTagCorrelationScenario's own + // seeding block for the rationale): the qualifying tid must be the thread that + // allocates the tracked instances - this thread allocates every LEAK_BUFFER + // chunk, including the lateChunk this scenario watches. + leakTid = JavaProfiler.getTid(); + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(LEAK_BUFFER_TEST_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedTidTrendSample0(LEAK_BUFFER_TEST_KLASS_ID, leakTid, epoch * 3, epoch); + } + hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; + } + + // Wait for the BFS thread to fully complete its own first pass - not just + // admitStaticFieldRoots()'s one-time sweep of LEAK_BUFFER's *current* contents (an earlier, + // narrower version of this wait checked exactly that, via a since-removed + // hasCompletedStaticFieldSweepForTest0() seam), but the WHOLE pass, static-field sweep and + // expandFrontier()/collectStaleExpandedEntriesForRotation() alike - strictly before this + // scenario creates the one chunk whose chain it is actually going to watch for. This is the + // entire point of this scenario: proving that an element appended to a static field's + // collection AFTER that field's List is already ReferenceChainTracker-EXPANDED gets discovered + // via a LATER re-expansion of that stale entry, not via the very first pass that admitted it. + // Waiting on only the static-field sub-step (found the hard way, by actually instrumenting + // runPassManualWalk() and reading cacheResolvedChain()'s own timestamp against it) is not + // enough: admitStaticFieldRoots() and expandFrontier() are two steps of the SAME runPass() + // call, so Java can observe the former's completion while the latter - which is what actually + // marks LEAK_BUFFER's List entry EXPANDED - is still in flight, still racing lateChunk's + // creation exactly the way the sleep-based version this replaced did. Waiting for + // referenceChainPassesRunForTest0() to advance past its pre-seeding snapshot instead only + // becomes true once that ENTIRE pass (both steps) has finished. Non-debug builds have no such + // seam and fall back to a generous sleep - acceptable there because the non-debug path never + // pins a specific representative anyway (see the debugBuild branch below): it just waits on + // real, much-slower allocation sampling to notice the growth on its own, so a few extra + // seconds of margin either way changes nothing about what the eventual match would prove. + if (debugBuild) { + boolean passCompleted = false; + // admitStaticFieldRoots() sweeps every loaded class in one FollowReferences call - a cold + // external JVM's full JUnit/JMC/Gradle-worker classpath (thousands of classes) can need + // several retries against pause-time-SLO pacing before one completes without truncating (see + // that method's own header comment on retry-from-scratch-but-cheap-on-already-admitted) - + // generous margin for the same reason ExternalProcessReferenceChainTest's own budget/ + // pausetarget are raised this far above the in-process test's defaults. + for (int i = 0; i < 300 && !passCompleted; i++) { + passCompleted = JavaProfiler.referenceChainPassesRunForTest0() != initialPasses; + if (!passCompleted) { + Thread.sleep(100); + } + } + if (!passCompleted) { + throw new IllegalStateException( + "ReferenceChainTracker never completed its first pass within 30s - cannot safely " + + "create the chunk this scenario is supposed to discover only via a later " + + "re-expansion of an already-EXPANDED static-field entry"); + } + } else { + Thread.sleep(3000); + } + // Snapshotted right after the wait above confirms the first pass has fully completed - any + // later change (advance, or reset-to-lower via a restart) proves a walk that started strictly + // after lateChunk existed. + int passesRunBeforeLateChunk = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; + byte[] lateChunk = new byte[CHUNK_BYTES]; + LEAK_BUFFER.add(lateChunk); + + // From here on, seeded incrementally, one epoch per round below, rather than as a single + // upfront burst - found the hard way: profiler.execute()'s start() resets + // Profiler::classMap()'s generation synchronously, but LivenessTracker::cleanup_table() only + // observes that new generation (and updates its own _last_class_map_generation) lazily, on the + // BFS thread's own first tick. A burst seeded entirely before that first tick lands in + // _klass_population, then gets wiped wholesale the moment that tick's class-map-generation + // mismatch fires (cleanup_table()'s own comment, livenessTracker.cpp). Seeding one fresh epoch + // per round instead is self-healing exactly the way the real leak this scenario mirrors is: + // LEAK_BUFFER keeps growing every round regardless of any one-time startup housekeeping, so + // however many rounds a wipe costs, the trend simply resumes accumulating from the next + // round's seed call. (Waiting for the first pass to fully complete above already makes this + // particular race very unlikely to matter by this point, but there is no reason to give up the + // self-healing margin now that it costs nothing.) + ReferenceChainAssertions.ChainMatch match = null; + int totalRounds = 25; + for (int round = 1; round <= totalRounds && match == null; round++) { + // Continues growing LEAK_BUFFER every round after lateChunk - mirrors the real leak's own + // continuous growth - but lateChunk itself (created above, strictly after the first sweep) + // stays the fixed target this scenario watches for throughout. + LEAK_BUFFER.add(new byte[CHUNK_BYTES]); + System.gc(); + + if (match == null && debugBuild) { + // Seeding/polling every round unconditionally (not gated on passesRun() having advanced) + // is load-bearing, found the hard way: gating this on the tracker's OWN progress creates a + // deadlock - ReferenceChainTracker::hasLeakSignal() (which authorizes both a search's first + // pass and any later restart once the search reaches a terminal state) reads + // LivenessTracker::selectLeakCandidates(), which needs a continuously-refreshed population + // trend to keep reporting this candidate; stopping that refresh mid-search (to wait on the + // search's own progress) can stall the search itself with no path back, since nothing else + // keeps re-authorizing it. This scenario's target - a growing static-field collection - + // legitimately produces a continuous real signal in production; seeding continuously here + // is what actually mirrors that, not an artifact to gate away. + // + // Same debug-only seeded-representative short-circuit as LeakingCacheScenario's own - + // watches lateChunk specifically (added after the static field's first sweep), not + // whatever the real allocation sampler happens to pick. See SEED_EPOCHS_FOR_HYSTERESIS's + // own comment for the minimum epoch count this needs - kept uncapped here (not stopped + // once that minimum is reached) specifically so a wipe that costs an early round or two + // still leaves enough remaining rounds to reach it, rather than exhausting a fixed budget + // of calls before ever getting there. + hysteresisEpoch++; + JavaProfiler.seedKlassPopulationSample0( + LEAK_BUFFER_TEST_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0( + LEAK_BUFFER_TEST_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, lateChunk); + JavaProfiler.pollReferenceChainTargets0(); + + // Only TRUST a match once passesRun() has genuinely changed since the snapshot taken right + // before lateChunk was created - either advanced (the same, never-restarted search ran a + // later pass) or reset to a lower value (restartSearch() ran, which only ever happens on a + // threadLoop tick strictly after that snapshot, since time only moves forward). Either way + // proves the match came from a walk that started after lateChunk existed, ruling out the + // one race this whole seam exists to rule out: the very first, still-in-flight pass finding + // lateChunk by accident because admitStaticFieldRoots() and expandFrontier() are two steps + // of that SAME call, and Java can observe the former's completion before the latter runs. + if (JavaProfiler.referenceChainPassesRunForTest0() != passesRunBeforeLateChunk) { + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + } + + if (match == null && !debugBuild) { + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + if (match == null && !debugBuild) { + Thread.sleep(300); + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + } + + // A cold external JVM's one-shot root-seeded walk over its whole reachable graph (full + // JUnit/JMC/Gradle-worker classpath) takes far longer than this scenario's own round loop - + // found the hard way: a first attempt at this scenario's tail wait (5 x 1s) gave up and + // printed NOT_FOUND_MARKER mere moments before the tracker's own runPass() actually + // completed (searchState=COMPLETED) and successfully built the chain. Mirrors + // ExternalProcessReferenceChainTest's own comment on why budget/pausetarget are raised this + // far for exactly this reason. + for (int attempt = 0; match == null && attempt < 40; attempt++) { + Thread.sleep(1500); + // Same passesRun()-advanced check as the round loop above (debug builds only) - by this + // point in a real run it has virtually always already flipped, but there is no reason to + // drop the guarantee here just because the round loop above didn't need it. + if (debugBuild && JavaProfiler.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { + continue; + } + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath); + } + + if (match == null) { + System.out.println(NOT_FOUND_MARKER); + return; + } + System.out.println(FOUND_MARKER + match.chain.get(0).getFullName()); + } + + private static ReferenceChainAssertions.ChainMatch findMatch(Path scratchDumpPath) throws Exception { + if (!Files.exists(scratchDumpPath)) { + return null; + } + IItemCollection events; + try (InputStream in = Files.newInputStream(scratchDumpPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + return ReferenceChainAssertions.findMatchForClass( + events.apply(ItemFilters.type("datadog.ReferenceChain")), byte[].class); + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java new file mode 100644 index 0000000000..f72d4fa1ec --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java @@ -0,0 +1,226 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.AbstractProcessProfilerTest; +import com.datadoghq.profiler.Platform; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +/** + * The thread-local taxonomy of the leak-tag correlation use case, end-to-end + * in a separate child JVM (same separate-process rationale as + * {@code LeakTagCorrelationReferenceChainTest}'s own class comment): a growing + * {@code byte[]} collection held ONLY through the leaking thread's + * {@code ThreadLocal} map must still produce a correlated + * {@code datadog.ReferenceChain} - {@code targetTag} in the leak-tag pool + * range, matched by a {@code datadog.HeapLiveObject} {@code leakTag}. The + * scenario is {@code ThreadLocalLeakScenario}; this test drives it and + * asserts the reported correlation, with a filtered child-log tail embedded + * in any failure message for diagnosability without a redeploy. + */ +@Tag("slow") +public class ThreadLocalLeakReferenceChainTest extends AbstractProcessProfilerTest { + + // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE) - the + // contract under test, asserted against the runtime event values the child + // reports. + private static final long LEAK_TAG_BASE = 0x40000000L; + private static final long LEAK_TAG_POOL_SIZE = 256; + + @Test + void shouldCorrelateThreadLocalHeldLeakChain() throws Exception { + assumeFalse(Platform.isJavaVersion(8)); + assumeFalse(Platform.isJ9()); + assumeFalse(Platform.isZing()); + + Path scratchDumpPath = Files.createTempFile("referencechains-tl-correlation-", ".jfr"); + Files.deleteIfExists(scratchDumpPath); + Path continuousJfrPath = Files.createTempFile("referencechains-tl-correlation-continuous-", ".jfr"); + Deque testLogTail = new ArrayDeque<>(); + try { + // Same raised budget/ratio rationale as + // LeakTagCorrelationReferenceChainTest (cold external JVM, ~100k filler, + // memory=64:l:1.0 keeping every chunk allocation tracked). + String startCommand = "start,memory=64:l:1.0,generations=true," + + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:" + + "pausetarget=60000" + + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); + String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); + + List jvmArgs = Collections.singletonList( + "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + + AtomicReference resultLine = new AtomicReference<>(); + LaunchResult result = launch("threadlocal-leak", jvmArgs, packedCommand, + Collections.emptyMap(), + 150, + line -> { + if (line.startsWith(ThreadLocalLeakScenario.FOUND_MARKER) + || line.equals(ThreadLocalLeakScenario.NOT_FOUND_MARKER) + || line.startsWith(ThreadLocalLeakScenario.TAG_OUT_OF_POOL_MARKER) + || line.startsWith(ThreadLocalLeakScenario.NO_LIVE_OBJECT_MARKER)) { + resultLine.set(line); + } + if (line.startsWith("[TEST::INFO]")) { + testLogTail.addLast(line); + while (testLogTail.size() > 2000) { + testLogTail.removeFirst(); + } + } + return LineConsumerResult.CONTINUE; + }, + null); + + assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); + assertNotNull(resultLine.get(), "Child process never printed a recognizable result " + + "marker on stdout" + diagnostics(testLogTail)); + String line = resultLine.get(); + if (line.startsWith(ThreadLocalLeakScenario.FOUND_MARKER)) { + long targetTag = Long.parseLong( + line.substring(ThreadLocalLeakScenario.FOUND_MARKER.length()).trim()); + assertTrue(targetTag >= LEAK_TAG_BASE && targetTag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE, + "Reported targetTag " + targetTag + " is outside the leak-tag pool range [" + + LEAK_TAG_BASE + ", " + (LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE) + ")"); + return; + } + fail("Thread-local leak-tag correlation did not succeed: " + line + + diagnostics(testLogTail)); + } finally { + Files.deleteIfExists(scratchDumpPath); + Files.deleteIfExists(continuousJfrPath); + } + } + + /** + * Same contract as {@link #shouldCorrelateThreadLocalHeldLeakChain()}, but + * the leak thread is started only AFTER the recording began. Regression + * test for the {@code Profiler::onThreadStart} → + * {@code registerThreadObject()} wiring: a mid-recording thread is + * invisible to {@code registerExistingThreads()}'s one-time snapshot, so + * only the onThreadStart registration lets + * {@code walkCandidateThreadLocals()} reach its ThreadLocalMap-held leak. + * With the wiring broken this child reports + * {@code ThreadLocalLeakScenario.NOT_FOUND_MARKER} and this test fails. + */ + @Test + void shouldCorrelateLeakOnThreadCreatedAfterRecordingStart() throws Exception { + assumeFalse(Platform.isJavaVersion(8)); + assumeFalse(Platform.isJ9()); + assumeFalse(Platform.isZing()); + + Path scratchDumpPath = Files.createTempFile("referencechains-tl-late-", ".jfr"); + Files.deleteIfExists(scratchDumpPath); + Path continuousJfrPath = Files.createTempFile("referencechains-tl-late-continuous-", ".jfr"); + Deque testLogTail = new ArrayDeque<>(); + try { + String startCommand = "start,memory=64:l:1.0,generations=true," + + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:" + + "pausetarget=60000" + + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); + String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); + + List jvmArgs = Collections.singletonList( + "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + + AtomicReference resultLine = new AtomicReference<>(); + LaunchResult result = launch("threadlocal-leak-late", jvmArgs, packedCommand, + Collections.emptyMap(), + 150, + line -> { + if (line.startsWith(ThreadLocalLeakScenario.FOUND_MARKER) + || line.equals(ThreadLocalLeakScenario.NOT_FOUND_MARKER) + || line.startsWith(ThreadLocalLeakScenario.TAG_OUT_OF_POOL_MARKER) + || line.startsWith(ThreadLocalLeakScenario.NO_LIVE_OBJECT_MARKER)) { + resultLine.set(line); + } + if (line.startsWith("[TEST::INFO]")) { + testLogTail.addLast(line); + while (testLogTail.size() > 2000) { + testLogTail.removeFirst(); + } + } + return LineConsumerResult.CONTINUE; + }, + null); + + assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); + assertNotNull(resultLine.get(), "Child process never printed a recognizable result " + + "marker on stdout" + diagnostics(testLogTail)); + String line = resultLine.get(); + assertTrue(line.startsWith(ThreadLocalLeakScenario.FOUND_MARKER), + "Thread-local leak held by a thread created after recording start was not " + + "correlated: " + line + diagnostics(testLogTail)); + } finally { + Files.deleteIfExists(scratchDumpPath); + Files.deleteIfExists(continuousJfrPath); + } + } + + /** + * A filtered summary of the child's TEST_LOG stream for failure messages - + * same first-things-to-check selection as + * LeakTagCorrelationReferenceChainTest.diagnostics(): the thread-walk and + * pass summaries (the new machinery this scenario exercises), the gotw + * batch-size trace, and any leak-tag interception or correlation lines. + */ + private static String diagnostics(Deque testLogTail) { + if (testLogTail.isEmpty()) { + return "\n(no TEST_LOG output captured - non-debug build or no reference-chain passes ran)"; + } + List threadWalk = new ArrayList<>(); + List runPassDone = new ArrayList<>(); + List gotw = new ArrayList<>(); + List leakTagLines = new ArrayList<>(); + for (String s : testLogTail) { + if (s.contains("walkCandidateThreadLocals")) { + threadWalk.add(s); + } else if (s.contains("runPass done:")) { + runPassDone.add(s); + } else if (s.contains("expandFrontier gotw")) { + gotw.add(s); + } else if (s.contains("leak-tag intercepted") + || s.contains("correlateAdmittedLeakTag") + || s.contains("tagLeakInstances summary")) { + leakTagLines.add(s); + } + } + StringBuilder sb = new StringBuilder("\n--- filtered TEST_LOG tail ---\n"); + appendTail(sb, "thread-walk:", threadWalk, 10); + appendTail(sb, "runPass:", runPassDone, 5); + appendTail(sb, "gotw:", gotw, 5); + appendTail(sb, "leak-tag:", leakTagLines, 10); + return sb.toString(); + } + + private static void appendTail(StringBuilder sb, String label, List lines, int cap) { + if (lines.isEmpty()) { + return; + } + sb.append(label).append('\n'); + int start = Math.max(0, lines.size() - cap); + for (int i = start; i < lines.size(); i++) { + sb.append(" ").append(lines.get(i)).append('\n'); + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java new file mode 100644 index 0000000000..b0f2e10ee2 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -0,0 +1,400 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.JavaProfiler; +import org.openjdk.jmc.common.IMCType; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.IType; +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * The thread-local taxonomy of the leak-tag correlation use case, run inside a + * separate child JVM by {@code ExternalLauncher}'s {@code threadlocal-leak} mode: + * a growing {@code byte[]} collection held ONLY through the leaking thread's + * own {@code ThreadLocal} map - never by any static field - must end up + * reported as a {@code datadog.ReferenceChain} event whose {@code targetTag} + * is a leak-tag-pool tag, matching a {@code datadog.HeapLiveObject} event's + * {@code leakTag} for the same object. + * + *

    This is the taxonomy shape {@code ReferenceChainTracker}'s thread-scoped + * walk exists for (walkCandidateThreadLocals(), referenceChains.cpp): the + * retained chunks are reachable only through the Thread object's + * {@code threadLocals} -> ThreadLocalMap -> table -> Entry -> value path, so + * the scenario proves that holder shape keeps working end-to-end - the + * thread-object registry, the anchor admission and the descend gates do not + * break the plain chain + correlation contract. The transient-root gate and + * the noise-chain suppression are {@code LeakTagCorrelationScenario}'s own + * contracts and are deliberately not re-asserted here. + * + *

    Same separate-process, seed-before-start, per-round hysteresis + * maintenance and debug-seam patterns as {@code LeakTagCorrelationScenario}'s + * own class comment (the one-shot root-seeded walk is a per-process + * resource, and a one-shot seeded ramp ages out of LivenessTracker's + * hysteresis once real fold samples interleave). + */ +public final class ThreadLocalLeakScenario { + private ThreadLocalLeakScenario() {} + + /** + * The leak sink: a per-thread growing list, touched ONLY by the leaking + * thread below. Deliberately NOT static-retained anywhere - the whole + * point is that the only durable path to the chunks runs through the + * leaking thread's ThreadLocalMap. + */ + private static final ThreadLocal> SINK = + ThreadLocal.withInitial(() -> new ArrayList<>()); + + /** Scale ingredient - same rationale as LeakTagCorrelationScenario.FILLER. */ + static final Object[][] FILLER = new Object[1500][]; + + // Distinct from the other scenarios' ids (987301-987304) - each scenario + // runs in its own child JVM, but a distinct id keeps logs unambiguous. + private static final int LEAK_KLASS_ID = 987305; + + // See LeakTagCorrelationScenario.SEED_EPOCHS_FOR_HYSTERESIS' own comment. + private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; + private static final int SEED_CHUNKS = 10; + + private static int hysteresisEpoch = 0; + + /** + * Chunk size that clears ObjectSampler's 256KiB sampling floor on its own + * (see LeakTagCorrelationScenario.CHUNK_BYTES' own comment): every chunk + * allocation must be sampled so it becomes a tracked, taggable + * live-heap instance. + */ + private static final int CHUNK_BYTES = 1_000_000; + + // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE). + private static final long LEAK_TAG_BASE = 0x40000000L; + private static final long LEAK_TAG_POOL_SIZE = 256; + + /** + * Handoff for the leak thread's profiler tid and one seed chunk - the + * main thread seeds LivenessTracker's per-(klass, tid) qualification with + * the REAL allocating thread's tid, and that tid lives on the leak thread. + * The chunk reference is dropped by take() so the main thread never keeps a + * second path to the sink alive. + */ + private static final class LeakHandoff { + volatile int tid; + volatile byte[] seedChunk; + + void publish(int tid, byte[] chunk) { + this.tid = tid; + this.seedChunk = chunk; + } + + byte[] take() { + byte[] chunk = seedChunk; + seedChunk = null; + return chunk; + } + + /** + * Bounded wait for the leak thread's first publish. Only the + * late-thread mode needs it: there the thread is started after the + * profiler (and thus after {@code profiler.execute()} returned), so the + * main thread cannot rely on the start-command's JNI work having + * scheduled the leak thread far enough to publish. + */ + boolean awaitPublication(long timeoutMs) throws InterruptedException { + long deadline = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(timeoutMs); + while (seedChunk == null && System.nanoTime() < deadline) { + Thread.sleep(10); + } + return seedChunk != null; + } + } + + /** Printed to stdout, followed by the correlated targetTag, on full success. */ + public static final String FOUND_MARKER = "[tl-correlation-found] "; + + /** Printed to stdout if no leaked byte[] chain was ever observed. */ + public static final String NOT_FOUND_MARKER = "[tl-correlation-not-found]"; + + /** Printed to stdout (with the tag) if a chain's targetTag is out of pool range. */ + public static final String TAG_OUT_OF_POOL_MARKER = "[tl-correlation-tag-out-of-pool] "; + + /** Printed to stdout (with the tag) if no HeapLiveObject matched the chain's tag. */ + public static final String NO_LIVE_OBJECT_MARKER = "[tl-correlation-no-live-object] "; + + private static final class ScanResult { + ReferenceChainAssertions.ChainMatch payloadChain; + boolean payloadChainInPool; + int liveObjectMatchesForChainTag; + } + + public static void run(JavaProfiler profiler, String startCommand, + Path scratchDumpPath) throws Exception { + run(profiler, startCommand, scratchDumpPath, false); + } + + /** + * @param lateThread when true, the leak thread is started only AFTER the + * recording began. This is the regression shape for + * {@code Profiler::onThreadStart}'s {@code registerThreadObject()} + * wiring (referenceChains.h): a thread born mid-recording is invisible + * to {@code registerExistingThreads()}'s one-time snapshot, so with + * the wiring broken the (klass, tid) qualification is seeded but + * {@code walkCandidateThreadLocals()} finds no Thread object for the + * tid and the ThreadLocalMap-held chain is never reached (the child + * prints {@link #NOT_FOUND_MARKER}). + */ + public static void run(JavaProfiler profiler, String startCommand, + Path scratchDumpPath, boolean lateThread) throws Exception { + // Build the whole fixture before the profiler starts - see the class + // comment and LeakingCacheScenario's seed-before-start rationale. + for (int row = 0; row < FILLER.length; row++) { + Object[] column = new Object[64]; + for (int col = 0; col < column.length; col++) { + column[col] = new FillerNode(); + } + FILLER[row] = column; + } + + LeakHandoff handoff = new LeakHandoff(); + // One round = the main thread offers a token; the leak thread takes it and + // appends one chunk to ITS OWN ThreadLocal sink (same persistent-allocator + // shape as ReferenceChainTrackingTest's togcroot fixture, so the chunks are + // always allocated on the same qualifying tid). + BlockingQueue rounds = new ArrayBlockingQueue<>(32); + Thread leakThread = new Thread(() -> { + List sink = SINK.get(); + for (int i = 0; i < SEED_CHUNKS; i++) { + sink.add(new byte[CHUNK_BYTES]); + } + handoff.publish(JavaProfiler.getTid(), sink.get(0)); + while (true) { + try { + if (rounds.poll(50, TimeUnit.MILLISECONDS) == null) { + continue; + } + sink.add(new byte[CHUNK_BYTES]); + } catch (InterruptedException e) { + return; + } + } + }, "threadlocal-leak"); + leakThread.setDaemon(true); + if (!lateThread) { + leakThread.start(); + } + + if (startCommand != null && !startCommand.isEmpty()) { + profiler.execute(startCommand); + } + + if (lateThread) { + leakThread.start(); + // The debug seeding below consumes the handoff immediately; the + // thread is only just started, so wait for its first publish. + if (!handoff.awaitPublication(30000)) { + System.out.println(NOT_FOUND_MARKER); + return; + } + } + + boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + int leakTid = 0; + if (debugBuild) { + // Representative first, seeds second - same aliasing rationale as + // LeakTagCorrelationScenario (the rep's real klass id must own the + // seeds or the seeds authorize nothing). The rep chunk stays durably + // reachable only through the leak thread's ThreadLocalMap - the + // take() above dropped this handoff's reference. + JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, handoff.take()); + leakTid = handoff.tid; + for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { + JavaProfiler.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); + JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); + } + // Wait for the BFS thread's first full pass, then poll - same + // load-bearing wait as LeakTagCorrelationScenario. + int initialPasses = JavaProfiler.referenceChainPassesRunForTest0(); + for (int i = 0; i < 300; i++) { + if (JavaProfiler.referenceChainPassesRunForTest0() != initialPasses) { + break; + } + Thread.sleep(100); + } + JavaProfiler.pollReferenceChainTargets0(); + } else { + Thread.sleep(5000); + } + + ScanResult best = null; + int totalRounds = 25; + for (int round = 1; round <= totalRounds; round++) { + if (!rounds.offer(round)) { + throw new IllegalStateException("rounds queue rejected round " + round); + } + if (debugBuild) { + hysteresisEpoch++; + JavaProfiler.seedKlassPopulationSample0( + LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + } + Thread.sleep(300); + System.gc(); + profiler.dump(scratchDumpPath); + // Scan every 3rd round - same scan-churn rationale as + // LeakTagCorrelationScenario's own comment. + if (round % 3 == 0) { + best = keepBest(best, scan(scratchDumpPath)); + if (best != null && best.payloadChainInPool + && best.liveObjectMatchesForChainTag > 0) { + break; + } + } + } + + for (int attempt = 0; attempt < 5; attempt++) { + if (best != null && best.payloadChainInPool && best.liveObjectMatchesForChainTag > 0) { + break; + } + if (debugBuild) { + hysteresisEpoch++; + JavaProfiler.seedKlassPopulationSample0( + LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); + JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + } + if (!rounds.offer(attempt)) { + throw new IllegalStateException( + "rounds queue rejected attempt " + attempt); + } + Thread.sleep(1000); + System.gc(); + profiler.dump(scratchDumpPath); + best = keepBest(best, scan(scratchDumpPath)); + } + + if (best == null || best.payloadChain == null) { + System.out.println(NOT_FOUND_MARKER); + return; + } + if (!best.payloadChainInPool) { + System.out.println(TAG_OUT_OF_POOL_MARKER + best.payloadChain.targetTag); + return; + } + if (best.liveObjectMatchesForChainTag == 0) { + System.out.println(NO_LIVE_OBJECT_MARKER + best.payloadChain.targetTag); + return; + } + System.out.println(FOUND_MARKER + best.payloadChain.targetTag); + } + + private static ScanResult keepBest(ScanResult best, ScanResult scan) { + if (scan == null) { + return best; + } + if (scan.payloadChain != null) { + return scan; + } + return best != null ? best : scan; + } + + /** + * Scans one dump: the first leaked {@code byte[]} chain (if any), whether + * its targetTag is in the leak-tag pool, and how many + * {@code datadog.HeapLiveObject} events for a leaked {@code byte[]} carry + * that same tag in {@code leakTag}. Simplified from + * LeakTagCorrelationScenario.scan() - no transient/noise gate scans (those + * are that scenario's own contracts). + */ + private static ScanResult scan(Path scratchDumpPath) throws Exception { + if (!Files.exists(scratchDumpPath)) { + return null; + } + IItemCollection events; + try (InputStream in = Files.newInputStream(scratchDumpPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + ScanResult result = new ScanResult(); + + IItemCollection chains = events.apply( + org.openjdk.jmc.common.item.ItemFilters.type("datadog.ReferenceChain")); + for (IItemIterable iterable : chains) { + IType type = iterable.getType(); + IMemberAccessor chainAccessor = + ReferenceChainAssertions.findAccessor(type, "chain"); + IMemberAccessor targetTagAccessor = + ReferenceChainAssertions.findAccessor(type, "targetTag"); + for (IItem item : iterable) { + Object chainValue = chainAccessor.getMember(item); + if (!(chainValue instanceof Object[])) { + continue; + } + Object[] rawChain = (Object[]) chainValue; + String leaf = rawChain.length > 0 && rawChain[0] instanceof IMCType + ? ((IMCType) rawChain[0]).getFullName() : ""; + long targetTag = targetTagAccessor != null + ? ReferenceChainAssertions.numberValue(targetTagAccessor.getMember(item)) : -1; + if (!"byte[]".equals(leaf)) { + continue; + } + boolean inPool = + targetTag >= LEAK_TAG_BASE && targetTag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE; + // Prefer the first in-pool (correlated) chain over any out-of-pool + // one seen earlier - same rationale as + // LeakTagCorrelationScenario.scan(). + if ((inPool && !result.payloadChainInPool) || result.payloadChain == null) { + List chain = new ArrayList<>(rawChain.length); + for (Object element : rawChain) { + chain.add((IMCType) element); + } + result.payloadChain = new ReferenceChainAssertions.ChainMatch(chain, targetTag, 0); + result.payloadChainInPool = inPool; + } + } + } + + if (result.payloadChainInPool) { + IItemCollection liveObjects = events.apply( + org.openjdk.jmc.common.item.ItemFilters.type("datadog.HeapLiveObject")); + for (IItemIterable iterable : liveObjects) { + IType type = iterable.getType(); + IMemberAccessor leakTagAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTag"); + IMemberAccessor objectClassAccessor = + ReferenceChainAssertions.findAccessor(type, "objectClass"); + for (IItem item : iterable) { + long leakTag = leakTagAccessor != null + ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; + if (leakTag != result.payloadChain.targetTag) { + continue; + } + Object objectClass = objectClassAccessor != null + ? objectClassAccessor.getMember(item) : null; + if (objectClass instanceof IMCType + && "byte[]".equals(((IMCType) objectClass).getFullName())) { + result.liveObjectMatchesForChainTag++; + } + } + } + } + return result; + } + + /** Scale-only object - same rationale as LeakTagCorrelationScenario.FillerNode. */ + static final class FillerNode { + long p0, p1, p2, p3, p4, p5, p6, p7; + } +} From 926cd9a337bb3fb3ac62564a16950692c11f35e2 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 12 Oct 2026 10:00:57 +0200 Subject: [PATCH 02/28] javaApi: re-baseline class-map generation in the population reset seam selectLeakCandidates() returns nothing while classMap()->generation() differs from the tracker's baseline, and that baseline is only re-synced by cleanup_table()'s GC-driven sweep. A test JVM typically runs several recordings before a seeding test, each start() bumping the generation, and no GC may land between start() and the seeded select call. After the reset seam's wipe no stale-namespace population data remains, so adopt the current generation. --- ddprof-lib/src/main/cpp/javaApi.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 904a7b6e37..a9e16b5119 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1173,6 +1173,17 @@ Java_com_datadoghq_profiler_JavaProfiler_resetKlassPopulationForTest0( JNIEnv *env, jclass unused) { ProfiledThread::initCurrentThreadSignalSafe(); LivenessTracker::instance()->klassPopulationResetForTest(); + // Re-baseline the class-map generation: selectLeakCandidates() returns + // nothing while classMap()->generation() differs from the tracker's + // baseline, and that baseline is only re-synced by cleanup_table()'s + // GC-driven sweep - but this JVM has typically already run several + // recordings (each Profiler::start() clearAll() bumps the generation) + // before this test's seeding, and no GC may land between start() and + // the seeded selectLeakCandidates() call. After the wipe above there is + // no stale-namespace population data left to protect, so adopting the + // current generation is safe. + LivenessTracker::instance()->classMapGenerationSetForTest( + Profiler::instance()->classMap()->generation()); } extern "C" DLLEXPORT jintArray JNICALL From 1db3753497932775937e93b12a42b343506905a3 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 12 Oct 2026 10:35:36 +0200 Subject: [PATCH 03/28] ReferenceChainJfrParserTest: skip when the gtest fixture is absent The roundtrip JFR fixture is produced by referenceChainJfrRoundtrip_ut, which does not run on CI runners without Google Test (e.g. the el7 testRelease job); asserting its existence failed the job there. Assume instead of assert so the test skips where the gtest artifact is unavailable and still executes wherever the gtest ran. --- .../referencechains/ReferenceChainJfrParserTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java index d33072a588..9017c3f587 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java @@ -5,6 +5,7 @@ package com.datadoghq.profiler.referencechains; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; import org.openjdk.jmc.common.IMCType; import org.openjdk.jmc.common.item.IAccessorKey; @@ -82,7 +83,13 @@ private static Path roundtripJfrPath() { public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() throws IOException, CouldNotLoadRecordingException { Path jfrPath = roundtripJfrPath(); - assertTrue(Files.exists(jfrPath), + // Assumption, not assertion: the fixture is produced by the companion gtest + // (referenceChainJfrRoundtrip_ut.cpp), which does not run on CI runners + // without Google Test (e.g. the el7 testRelease job) - there the file can + // never exist and this test would fail spuriously. Where the gtest ran in + // the same Gradle invocation (local testDebug, gtest-equipped CI jobs), the + // file exists and the test executes; otherwise it is skipped. + Assumptions.assumeTrue(Files.exists(jfrPath), "Expected " + jfrPath + " to exist - run " + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " + "(referenceChainJfrRoundtrip_ut.cpp produces this file)."); From 5a32be503348b40bea9d395d7122b6bdf87b3bc1 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 13 Oct 2026 13:00:04 +0200 Subject: [PATCH 04/28] liveness: invalidate cached class IDs when adopting a class-map generation classMapGenerationSetForTest() bypassed cleanup_table()'s class-map-reset branch, so surviving liveness entries kept cached_klass_id values resolved against the previous StringDictionary namespace and could alias an unrelated class after a recording restart. Zero the cached IDs when the generation actually changes (fixes autotest finding on PR #800). --- ddprof-lib/src/main/cpp/javaApi.cpp | 7 +++++-- ddprof-lib/src/main/cpp/livenessTracker.h | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index a9e16b5119..1ac0e7b7d8 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1180,8 +1180,11 @@ Java_com_datadoghq_profiler_JavaProfiler_resetKlassPopulationForTest0( // recordings (each Profiler::start() clearAll() bumps the generation) // before this test's seeding, and no GC may land between start() and // the seeded selectLeakCandidates() call. After the wipe above there is - // no stale-namespace population data left to protect, so adopting the - // current generation is safe. + // no stale-namespace population data left, and + // classMapGenerationSetForTest() itself invalidates every tracking entry's + // cached_klass_id when the generation changes (the same thing + // cleanup_table()'s reset branch does), so adopting the current + // generation is safe. LivenessTracker::instance()->classMapGenerationSetForTest( Profiler::instance()->classMap()->generation()); } diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index a10fb017cc..84fce37cff 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -1457,8 +1457,18 @@ class alignas(alignof(SpinLock)) LivenessTracker { } // Lets a test that seeds _klass_population directly (no cleanup_table() // sweep) mark it as belonging to the given class-map generation. + // Adopting a new generation here bypasses cleanup_table()'s class-map-reset + // branch, so it must do that branch's tracking-table half itself: every + // cached_klass_id was resolved against the old StringDictionary namespace + // and could alias an unrelated class in the new one. (_klass_population is + // the caller's job - see klassPopulationResetForTest().) void classMapGenerationSetForTest(u64 generation) { _table_lock.lock(); + if (generation != _last_class_map_generation && _table != NULL) { + for (u32 i = 0; i < (u32)_table_size; i++) { + _table[i].cached_klass_id = 0; + } + } _last_class_map_generation = generation; _table_lock.unlock(); } From 29a8459663c2dba3ce30e2633fe4f77e49d90732 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 13 Oct 2026 13:00:13 +0200 Subject: [PATCH 05/28] reference-chains: frontier-state and resolved-tag test seams Add getReferenceChainFrontierStateForTest0() and getReferenceChainResolvedTargetTagForTest0() so tests synchronize on an object's actual frontier expansion and on the CURRENT search's resolved chain target (source_search_ns checked, so a restart-reused tag cannot match a stale chain) instead of inferring progress from pass counters. Release/no-JVMTI variants get stubbed natives. --- ddprof-lib/src/main/cpp/javaApi.cpp | 44 +++++++++++++++++++ .../src/main/cpp/referenceChainEvents.cpp | 17 +++++++ ddprof-lib/src/main/cpp/referenceChains.cpp | 11 +++++ ddprof-lib/src/main/cpp/referenceChains.h | 16 +++++++ .../com/datadoghq/profiler/JavaProfiler.java | 15 +++++++ 5 files changed, 103 insertions(+) diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 1ac0e7b7d8..36b55c5370 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1287,6 +1287,38 @@ Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingSizeForTest0( return (jlong)ReferenceChainTracker::instance()->pendingExpandSizeForTest(); } +// Reads target's existing tag without tagging it; see frontierStateForTest(). +extern "C" DLLEXPORT jint JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainFrontierStateForTest0( + JNIEnv *env, jclass unused, jobject target) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr || target == nullptr) { + return -1; + } + jlong tag = 0; + if (jvmti->GetTag(target, &tag) != JVMTI_ERROR_NONE) { + return -1; + } + return (jint)ReferenceChainTracker::instance()->frontierStateForTest(tag); +} + +// Reads target's existing tag without tagging it; see resolvedChainTargetTagForTest(). +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainResolvedTargetTagForTest0( + JNIEnv *env, jclass unused, jobject target) { + ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (jvmti == nullptr || target == nullptr) { + return 0; + } + jlong tag = 0; + if (jvmti->GetTag(target, &tag) != JVMTI_ERROR_NONE) { + return 0; + } + return ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(tag); +} + // Seeds one heap-floor sample for secondsToOOM() without waiting for a GC. // Timestamps are only compared with each other, so any increasing sequence works. extern "C" DLLEXPORT void JNICALL @@ -1403,6 +1435,18 @@ Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingSizeForTest0( return 0; } +extern "C" DLLEXPORT jint JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainFrontierStateForTest0( + JNIEnv *env, jclass unused, jobject target) { + return -1; +} + +extern "C" DLLEXPORT jlong JNICALL +Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainResolvedTargetTagForTest0( + JNIEnv *env, jclass unused, jobject target) { + return 0; +} + extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_heapFloorRecordForTest0( JNIEnv *env, jclass unused, jlong usedBytes, jlong timestampNs) {} diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index de60c15403..40244eeee2 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -499,6 +499,23 @@ void ReferenceChainTracker::invalidateResolvedChain(jlong source_tag) { } } +jlong ReferenceChainTracker::resolvedChainTargetTagForTest(jlong object_tag) { + if (object_tag <= 0 || isLeakTag(object_tag)) { + return 0; + } + const u64 current_search_ns = load(_search_start_ns); + if (current_search_ns == 0) { + return 0; + } + ExclusiveLockGuard guard(&_resolved_chains_lock); + auto it = _resolved_chains.find(object_tag); + if (it == _resolved_chains.end() || + it->second.source_search_ns != current_search_ns) { + return 0; + } + return (jlong)it->second.event._target_tag; +} + int ReferenceChainTracker::snapshotResolvedChainTags(jlong *out) { if (out == nullptr) { return 0; diff --git a/ddprof-lib/src/main/cpp/referenceChains.cpp b/ddprof-lib/src/main/cpp/referenceChains.cpp index c755b119e1..6115831e19 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.cpp +++ b/ddprof-lib/src/main/cpp/referenceChains.cpp @@ -1063,6 +1063,17 @@ long ReferenceChainTracker::pendingExpandPositionForTest(jlong tag) const { return -1; } +int ReferenceChainTracker::frontierStateForTest(jlong object_tag) { + if (object_tag <= 0 || isLeakTag(object_tag) || _frontier == nullptr) { + return -1; + } + FrontierEntry entry{}; + if (!_frontier->lookup(object_tag, &entry)) { + return -1; + } + return (int)entry.state; +} + size_t ReferenceChainTracker::pendingExpandSizeForTest() const { MutexLocker engine_guard(_engine_lock); return _pending_expand.size() + _priority_expand.size(); diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index 6e1da0dac8..c286e2c22c 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -1368,6 +1368,22 @@ class ReferenceChainTracker { // for computing a position's fraction of the current backlog. size_t pendingExpandSizeForTest() const; + // Test seam - not part of the production API. The FrontierEntryState of the current search's + // frontier entry for the object carrying JVMTI tag `object_tag` (FRONTIER/EXPANDED/ABANDONED), + // or -1 if there is no such entry: untagged, leak-tagged (not yet intercepted by the walk), or + // no frontier table / slot. Lets a test synchronize on an object's actual expansion rather than + // inferring it from pass counters. + int frontierStateForTest(jlong object_tag); + + // Test seam - not part of the production API. Returns the targetTag a datadog.ReferenceChain + // event for the object currently carrying JVMTI tag `object_tag` would report, but only if the + // CURRENT search has already resolved and cached that object's own chain (_resolved_chains entry + // keyed by its frontier tag with source_search_ns == _search_start_ns). 0 otherwise: untagged, + // leak-tagged but not yet intercepted, no cached chain, or a chain cached by an earlier search + // generation (restartSearch() keeps _resolved_chains and reuses tags from 1, so raw tag equality + // alone could match a stale, unrelated chain). + jlong resolvedChainTargetTagForTest(jlong object_tag); + // Test seam - not part of the production API. Evaluates the real private scheduling gate without // executing a BFS pass. This is not read-only: while holding _engine_lock like threadLoop(), the // gate may account a completed search's pain and restart/reset terminal search state. It also diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 6f5bd6374b..a876264a24 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -671,6 +671,21 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l */ public static native long getReferenceChainPendingSizeForTest0(); + /** + * Test seam (debug native builds only, returns -1 in release): does not tag {@code target}. + * The state of the current search's frontier entry for target - 0 = FRONTIER (admitted, not + * yet expanded), 1 = EXPANDED, 3 = ABANDONED - or {@code -1} if target has no frontier entry + * (never admitted, or still carrying only a leak tag). + */ + public static native int getReferenceChainFrontierStateForTest0(Object target); + + /** + * Test seam (debug native builds only, returns 0 in release): does not tag {@code target}. + * The targetTag the {@code datadog.ReferenceChain} event for target carries, if the current + * search has already resolved and cached target's own chain; 0 otherwise. + */ + public static native long getReferenceChainResolvedTargetTagForTest0(Object target); + /** * Test seam (debug native builds only): seeds one heap-floor-ring sample - the input to * {@code LivenessTracker::secondsToOOM()}'s time-to-OOM projection - directly, bypassing the From bca633fc8564c26fe3e8c34263d6eb15b5197502 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 13 Oct 2026 13:00:20 +0200 Subject: [PATCH 06/28] reference-chains: harden leak-scenario synchronization per autotest review - ExternalProcess/LeakTagCorrelation: gate the frontier-rotation assertion on debug-instrumented builds (assumeTrue) and enable DD_PROFILING_REFERENCE_CHAINS_DEBUG=2 in the child when required - ThreadLocalLeakScenario: await handoff publication in both thread-start modes; keep the sink reachable only via the ThreadLocal and require ThreadLocalMap traversal in the reported chain - LeakTagCorrelationScenario: continue the seeded ramp during maintenance (hysteresisEpoch) and merge suppression violations across scans in keepBest() - StaticFieldGrowingCollectionScenario: wait for EXPANDED frontier state before creating the late chunk and pin the match to the late chunk's own resolved chain, not any byte[] chain - AggressiveLeakReferenceChainTest: capture the pass baseline before publishing heap-floor urgency --- .../AggressiveLeakReferenceChainTest.java | 4 +- .../ExternalProcessReferenceChainTest.java | 8 +- .../LeakTagCorrelationReferenceChainTest.java | 23 ++- .../LeakTagCorrelationScenario.java | 37 +++- .../ReferenceChainAssertions.java | 20 +- .../StaticFieldGrowingCollectionScenario.java | 185 +++++++++++++----- .../ThreadLocalLeakScenario.java | 70 +++++-- 7 files changed, 270 insertions(+), 77 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java index 8cc7f9a440..bcc7f9c809 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java @@ -86,6 +86,9 @@ public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() JavaProfiler.resetKlassPopulationForTest0(); JavaProfiler.resetReferenceChainSearchForTest0(); try { + // Baseline BEFORE publishing any urgency input: the live BFS thread may run the one + // urgency-authorized search as soon as the samples below land, and that pass must count. + int passesBefore = JavaProfiler.referenceChainPassesRunForTest0(); // 1000 MiB fake max heap, rising from 100 MiB to 991 MiB over 9 (fake) seconds: ~99 MiB/s, // 9 MiB of headroom left at the last sample -> projected time-to-OOM ~= 9/99 s, far under // OOM_URGENT_THRESHOLD_S = 300s. @@ -101,7 +104,6 @@ public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() "This test's own precondition: no per-klass candidate should exist, so passes running " + "below can only come from the aggregate urgent-OOM bypass"); - int passesBefore = JavaProfiler.referenceChainPassesRunForTest0(); // Idle BFS cadence is ~1s/pass; urgency ramps it to ~10ms after the latch. 20s covers the // first evaluation landing up to one idle cadence after the seeding loop. long deadline = System.currentTimeMillis() + 20_000; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java index cb91ad34f8..8e098e757f 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -12,6 +12,7 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -166,8 +167,11 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); - List jvmArgs = Collections.singletonList( - "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + // --add-opens: the scenario reflects into Collections$SynchronizedList.list and + // ArrayList.elementData to check its holders' expansion state before creating lateChunk. + List jvmArgs = Arrays.asList( + "-Dddprof_test.config=" + System.getProperty("ddprof_test.config"), + "--add-opens=java.base/java.util=ALL-UNNAMED"); AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("leak-static-field", jvmArgs, packedCommand, diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java index 656be82fa1..462a07c481 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -25,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * End-to-end coverage of the leak-tag correlation use case in a genuinely separate child @@ -94,13 +96,25 @@ private void assertCorrelationFound(String pauseTargetOption) throws Exception { * leak-accumulation rotation's FRONTIER-state selection is additionally asserted, and * the correlation result markers that the pod regime legitimately produces without the * full correlation completing (not-found, tag-out-of-pool - see the tight variant's own - * comment) are accepted instead of failed. + * comment) are accepted instead of failed. That FRONTIER signal is a level-2 + * {@code TEST_LOG} line, so this variant runs only on DEBUG-instrumented native builds and + * forces reference-chains debug level 2 in the child; the loose variant does not depend on + * logging and runs on every build. */ private void assertCorrelationFound(String pauseTargetOption, boolean requireFrontierRotation) throws Exception { assumeFalse(Platform.isJavaVersion(8)); assumeFalse(Platform.isJ9()); assumeFalse(Platform.isZing()); + if (requireFrontierRotation) { + // The FRONTIER-rotation signal is a level-2 TEST_LOG line (referenceChainWalk.cpp, + // collectLeakAccumulationCandidatesForRotation) - compiled out without -DDEBUG + // (debug/asan/tsan only) and silent unless DD_PROFILING_REFERENCE_CHAINS_DEBUG=2. + String config = System.getProperty("ddprof_test.config"); + assumeTrue(config != null && ("debug".equals(config) || config.endsWith("san")), + "FRONTIER-rotation assertion needs the reference-chains TEST_LOG stream, only " + + "compiled into DEBUG-instrumented builds (debug/asan/tsan)"); + } Path scratchDumpPath = Files.createTempFile("referencechains-correlation-", ".jfr"); Files.deleteIfExists(scratchDumpPath); @@ -134,8 +148,13 @@ private void assertCorrelationFound(String pauseTargetOption, "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); AtomicReference resultLine = new AtomicReference<>(); + // Level 2 is required for the FRONTIER-selection TEST_LOG line (rcDebugLevel.h); the + // loose variant keeps the default (silent) level. + Map childEnv = requireFrontierRotation + ? Collections.singletonMap("DD_PROFILING_REFERENCE_CHAINS_DEBUG", "2") + : Collections.emptyMap(); LaunchResult result = launch("leak-correlation", jvmArgs, packedCommand, - Collections.emptyMap(), + childEnv, // The scenario's own round loop (up to 25 rounds plus a 5-attempt grace period, // ~40-60s observed) on top of a cold external JVM's startup/classloading cost and // this scenario's larger graph - same margin logic as the existing scenarios. diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index f83eb16c15..7b3b3b3e14 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -185,7 +185,10 @@ NoisePayload take() { */ public static final String NOISE_CHAIN_MARKER = "[correlation-noise-chain-emitted] "; - /** One dump's scan outcome - all fields per the latest dump only. */ + /** + * Scan outcome: payload fields from one dump; transientViolations/noiseChains accumulated + * across all scans by keepBest(). + */ private static final class ScanResult { ReferenceChainAssertions.ChainMatch payloadChain; boolean payloadChainInPool; @@ -273,6 +276,9 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); JavaProfiler.seedTidTrendSample0(NOISE_KLASS_ID, noiseTid, epoch * 3, epoch); } + // Maintenance continues the seeded ramp (next epoch = SEED_EPOCHS_FOR_HYSTERESIS + 1) + // instead of restarting at 1 - same as StaticFieldGrowingCollectionScenario. + hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; // Wait for the BFS thread to fully complete its first pass (static sweep AND frontier // expansion) before any growth round - same load-bearing wait and rationale as // StaticFieldGrowingCollectionScenario: the late chunks added afterward must be @@ -293,7 +299,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // Each dump re-emits every resolved chain (drainPendingChainEvents()'s // snapshot-and-keep contract), so a later dump's scan is a superset state - the - // latest scan with a payload chain observed is always the best evidence. + // latest scan with a payload chain observed is always the best payload evidence. Gate + // violations are NOT taken from the latest scan alone: keepBest() accumulates them + // across every scan (a violating resolved chain may be invalidated/evicted before a + // later dump, and the gates are 'ever emitted' contracts). ScanResult best = null; int totalRounds = 25; // Slow growth per round is deliberate: each chunk clears the allocation @@ -377,15 +386,31 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD System.out.println(FOUND_MARKER + best.payloadChain.targetTag); } - /** Latest scan wins; a scan without a payload chain never displaces one that had it. */ + /** + * Payload state comes from the latest scan that saw a payload chain (a scan without one + * never displaces one that had it). Gate violations (transient/noise chains) are + * accumulated across ALL scans - they are 'ever emitted' contracts, and a violating + * resolved chain may be invalidated/evicted before a later dump. + */ private static ScanResult keepBest(ScanResult best, ScanResult scan) { if (scan == null) { return best; } - if (scan.payloadChain != null) { - return scan; + ScanResult result = scan.payloadChain != null || best == null ? scan : best; + ScanResult other = result == scan ? best : scan; + if (other != null) { + mergeDistinct(result.transientViolations, other.transientViolations); + mergeDistinct(result.noiseChains, other.noiseChains); + } + return result; + } + + private static void mergeDistinct(List into, List from) { + for (String s : from) { + if (!into.contains(s)) { + into.add(s); + } } - return best != null ? best : scan; } /** diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java index 447a35cfbf..a8df2e4b0b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java @@ -70,6 +70,21 @@ public static final class JfrChainMatch { * own comment). Returns {@code null} if {@code events} is empty or none match. */ public static ChainMatch findMatchForClass(IItemCollection events, Class targetClass) { + return findMatch(events, targetClass, false, 0); + } + + /** + * Same as {@link #findMatchForClass(IItemCollection, Class)}, but identifies one specific + * instance: only an item whose {@code targetTag} equals {@code requiredTargetTag} matches (an + * item without a readable {@code targetTag} never does). + */ + public static ChainMatch findMatchForClassAndTargetTag(IItemCollection events, + Class targetClass, long requiredTargetTag) { + return findMatch(events, targetClass, true, requiredTargetTag); + } + + private static ChainMatch findMatch(IItemCollection events, Class targetClass, + boolean requireTag, long requiredTargetTag) { if (events == null || !events.hasItems()) { return null; } @@ -94,11 +109,14 @@ public static ChainMatch findMatchForClass(IItemCollection events, Class targ || !targetName.equals(((IMCType) rawChain[0]).getFullName())) { continue; } + long targetTag = targetTagAccessor != null ? numberValue(targetTagAccessor.getMember(item)) : -1; + if (requireTag && targetTag != requiredTargetTag) { + continue; + } List chain = new ArrayList<>(rawChain.length); for (Object element : rawChain) { chain.add((IMCType) element); } - long targetTag = targetTagAccessor != null ? numberValue(targetTagAccessor.getMember(item)) : -1; int depth = depthAccessor != null ? (int) numberValue(depthAccessor.getMember(item)) : -1; return new ChainMatch(chain, targetTag, depth); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java index a1251f5fed..6f284d9f17 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java @@ -11,11 +11,14 @@ import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; import java.io.InputStream; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; /** * Mimics the real leak shape found in the {@code prof-analyzer-hotdog-jb} pod's leak generator @@ -40,9 +43,12 @@ private StaticFieldGrowingCollectionScenario() {} /** * Mirrors production's {@code ProfileAnalyzer.LEAK_BUFFER}: a static field holding a * {@code synchronizedList}, appended to (never reassigned) for the process's whole - * life. + * life. The backing ArrayList is pre-sized so it never reallocates during the scenario: the + * elementData array checked as EXPANDED before lateChunk is appended is the very array that ends + * up holding lateChunk. */ - static final List LEAK_BUFFER = Collections.synchronizedList(new ArrayList<>()); + static final List LEAK_BUFFER = + Collections.synchronizedList(new ArrayList<>(StaticFieldGrowingCollectionScenario.LEAK_BUFFER_CAPACITY)); // Arbitrary, scenario-chosen klass id - see LeakingCacheScenario.CACHED_PAYLOAD_TEST_KLASS_ID's // own comment; distinct from that scenario's id since both may run in the same suite (each in @@ -61,6 +67,15 @@ private StaticFieldGrowingCollectionScenario() {} private static final int SEED_CHUNKS = 10; + private static final int TOTAL_ROUNDS = 25; + + // Seeds + lateChunk + one per round: LEAK_BUFFER's elementData never reallocates. + private static final int LEAK_BUFFER_CAPACITY = SEED_CHUNKS + 1 + TOTAL_ROUNDS; + + // FrontierEntryState::EXPANDED (referenceChainFrontier.h), as reported by + // JavaProfiler.getReferenceChainFrontierStateForTest0(). + private static final int FRONTIER_STATE_EXPANDED = 1; + /** Printed to stdout, followed by the matched leaf class's name, on success. */ public static final String FOUND_MARKER = "[chain-found] "; @@ -71,7 +86,7 @@ private StaticFieldGrowingCollectionScenario() {} * Seeds {@link #LEAK_BUFFER}, starts the profiler, then keeps appending new {@code byte[]} * chunks to the same list instance - the exact shape implicated in the live hotdog-pod stall - * watching specifically for a chain to a chunk appended *after* the static field's one-time - * sweep, until a {@code datadog.ReferenceChain} event for it appears or {@code totalRounds} is + * sweep, until a {@code datadog.ReferenceChain} event for it appears or {@code TOTAL_ROUNDS} is * exhausted. */ public static void run(JavaProfiler profiler, String startCommand, Path scratchDumpPath) throws Exception { @@ -113,51 +128,68 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; } - // Wait for the BFS thread to fully complete its own first pass - not just - // admitStaticFieldRoots()'s one-time sweep of LEAK_BUFFER's *current* contents (an earlier, - // narrower version of this wait checked exactly that, via a since-removed - // hasCompletedStaticFieldSweepForTest0() seam), but the WHOLE pass, static-field sweep and - // expandFrontier()/collectStaleExpandedEntriesForRotation() alike - strictly before this + // Wait until LEAK_BUFFER's holders are ReferenceChainTracker-EXPANDED - strictly before this // scenario creates the one chunk whose chain it is actually going to watch for. This is the // entire point of this scenario: proving that an element appended to a static field's - // collection AFTER that field's List is already ReferenceChainTracker-EXPANDED gets discovered - // via a LATER re-expansion of that stale entry, not via the very first pass that admitted it. - // Waiting on only the static-field sub-step (found the hard way, by actually instrumenting - // runPassManualWalk() and reading cacheResolvedChain()'s own timestamp against it) is not - // enough: admitStaticFieldRoots() and expandFrontier() are two steps of the SAME runPass() - // call, so Java can observe the former's completion while the latter - which is what actually - // marks LEAK_BUFFER's List entry EXPANDED - is still in flight, still racing lateChunk's - // creation exactly the way the sleep-based version this replaced did. Waiting for - // referenceChainPassesRunForTest0() to advance past its pre-seeding snapshot instead only - // becomes true once that ENTIRE pass (both steps) has finished. Non-debug builds have no such - // seam and fall back to a generous sleep - acceptable there because the non-debug path never - // pins a specific representative anyway (see the debugBuild branch below): it just waits on - // real, much-slower allocation sampling to notice the growth on its own, so a few extra - // seconds of margin either way changes nothing about what the eventual match would prove. + // collection AFTER that field's List is already EXPANDED gets discovered via a LATER + // re-expansion of that stale entry, not via the very first pass that admitted it. + // admitStaticFieldRoots() and expandFrontier() are two steps of the SAME runPass() call, and + // neither the static-field sweep's completion nor a changed pass counter proves the holders + // were expanded (see the debug branch below), so the wait reads their frontier state directly. + // Non-debug builds have no such seam and fall back to a generous sleep - acceptable there + // because the non-debug path never pins a specific representative anyway (see the debugBuild + // branch below): it just waits on real, much-slower allocation sampling to notice the growth + // on its own, so a few extra seconds of margin either way changes nothing about what the + // eventual match would prove. if (debugBuild) { - boolean passCompleted = false; - // admitStaticFieldRoots() sweeps every loaded class in one FollowReferences call - a cold - // external JVM's full JUnit/JMC/Gradle-worker classpath (thousands of classes) can need - // several retries against pause-time-SLO pacing before one completes without truncating (see - // that method's own header comment on retry-from-scratch-but-cheap-on-already-admitted) - - // generous margin for the same reason ExternalProcessReferenceChainTest's own budget/ - // pausetarget are raised this far above the in-process test's defaults. - for (int i = 0; i < 300 && !passCompleted; i++) { - passCompleted = JavaProfiler.referenceChainPassesRunForTest0() != initialPasses; - if (!passCompleted) { + // A changed pass counter alone is not proof that LEAK_BUFFER's holders were expanded: + // truncated passes also bump it, and admitStaticFieldRoots() only processes a bounded + // chunk of classes per call. So the wait synchronizes on the holders' ACTUAL frontier state + // instead: the SynchronizedList, its backing ArrayList and that ArrayList's elementData + // array must all be EXPANDED before lateChunk exists - only then can lateChunk be reached + // exclusively via a later re-expansion. The pass-counter check stays as a precondition + // (and gates the reflective holder reads until the first pass - with its root + // enumeration - has finished). + Field backingListField; + Field elementDataField; + try { + backingListField = + Class.forName("java.util.Collections$SynchronizedList").getDeclaredField("list"); + backingListField.setAccessible(true); + elementDataField = ArrayList.class.getDeclaredField("elementData"); + elementDataField.setAccessible(true); + } catch (ReflectiveOperationException | RuntimeException e) { + throw new IllegalStateException("cannot reach LEAK_BUFFER's backing ArrayList/elementData" + + " - run with --add-opens java.base/java.util=ALL-UNNAMED", e); + } + // admitStaticFieldRoots() sweeps every loaded class - a cold external JVM's full + // JUnit/JMC/Gradle-worker classpath (thousands of classes) can need several retries + // against pause-time-SLO pacing, and the expansion backlog ahead of LEAK_BUFFER's holders + // is FIFO - generous margin for the same reason ExternalProcessReferenceChainTest's own + // budget/pausetarget are raised this far above the in-process test's defaults. Wall-clock + // deadline: the seam reads can contend with a running pass's locks. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(45); + boolean holdersExpanded = false; + while (!holdersExpanded && System.nanoTime() < deadline) { + holdersExpanded = JavaProfiler.referenceChainPassesRunForTest0() != initialPasses + && leakBufferHoldersExpanded(backingListField, elementDataField); + if (!holdersExpanded) { Thread.sleep(100); } } - if (!passCompleted) { + if (!holdersExpanded) { throw new IllegalStateException( - "ReferenceChainTracker never completed its first pass within 30s - cannot safely " - + "create the chunk this scenario is supposed to discover only via a later " - + "re-expansion of an already-EXPANDED static-field entry"); + "LEAK_BUFFER's holders never all reached EXPANDED within 45s (passesRun=" + + JavaProfiler.referenceChainPassesRunForTest0() + ", initial=" + initialPasses + + ", frontier states [list, backing ArrayList, elementData]=" + + describeHolderStates(backingListField, elementDataField) + + ") - cannot safely create the chunk this scenario is supposed to discover only " + + "via a later re-expansion of an already-EXPANDED static-field entry"); } } else { Thread.sleep(3000); } - // Snapshotted right after the wait above confirms the first pass has fully completed - any + // Snapshotted right after the wait above confirms the holders are EXPANDED - any // later change (advance, or reset-to-lower via a restart) proves a walk that started strictly // after lateChunk existed. int passesRunBeforeLateChunk = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; @@ -178,8 +210,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // particular race very unlikely to matter by this point, but there is no reason to give up the // self-healing margin now that it costs nothing.) ReferenceChainAssertions.ChainMatch match = null; - int totalRounds = 25; - for (int round = 1; round <= totalRounds && match == null; round++) { + for (int round = 1; round <= TOTAL_ROUNDS && match == null; round++) { // Continues growing LEAK_BUFFER every round after lateChunk - mirrors the real leak's own // continuous growth - but lateChunk itself (created above, strictly after the first sweep) // stays the fixed target this scenario watches for throughout. @@ -221,20 +252,29 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // one race this whole seam exists to rule out: the very first, still-in-flight pass finding // lateChunk by accident because admitStaticFieldRoots() and expandFrontier() are two steps // of that SAME call, and Java can observe the former's completion before the latter runs. + // + // The match is pinned to lateChunk's OWN chain: any byte[] chain (the seed chunks created + // before the first pass, discovered instances, other representatives) is not enough. The + // tag is read BEFORE dumping - the dump snapshots the same resolved-chain cache - and a + // round where the current search has not resolved lateChunk's chain yet is skipped. if (JavaProfiler.referenceChainPassesRunForTest0() != passesRunBeforeLateChunk) { - profiler.dump(scratchDumpPath); - match = findMatch(scratchDumpPath); + long lateChunkTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0(lateChunk); + if (lateChunkTargetTag != 0) { + profiler.dump(scratchDumpPath); + match = findMatch(scratchDumpPath, lateChunkTargetTag); + } } } + // Non-debug builds have no identity seam: class-only matching, as before. if (match == null && !debugBuild) { profiler.dump(scratchDumpPath); - match = findMatch(scratchDumpPath); + match = findMatch(scratchDumpPath, 0); } if (match == null && !debugBuild) { Thread.sleep(300); profiler.dump(scratchDumpPath); - match = findMatch(scratchDumpPath); + match = findMatch(scratchDumpPath, 0); } } @@ -250,11 +290,19 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // Same passesRun()-advanced check as the round loop above (debug builds only) - by this // point in a real run it has virtually always already flipped, but there is no reason to // drop the guarantee here just because the round loop above didn't need it. - if (debugBuild && JavaProfiler.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { - continue; + // Same lateChunk-identity pinning as the round loop above. + long requiredTargetTag = 0; + if (debugBuild) { + if (JavaProfiler.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { + continue; + } + requiredTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0(lateChunk); + if (requiredTargetTag == 0) { + continue; + } } profiler.dump(scratchDumpPath); - match = findMatch(scratchDumpPath); + match = findMatch(scratchDumpPath, requiredTargetTag); } if (match == null) { @@ -264,7 +312,12 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD System.out.println(FOUND_MARKER + match.chain.get(0).getFullName()); } - private static ReferenceChainAssertions.ChainMatch findMatch(Path scratchDumpPath) throws Exception { + /** + * {@code requiredTargetTag == 0}: the first byte[] chain (class-only, non-debug builds). + * Otherwise: only the byte[] chain whose targetTag identifies that one instance. + */ + private static ReferenceChainAssertions.ChainMatch findMatch(Path scratchDumpPath, + long requiredTargetTag) throws Exception { if (!Files.exists(scratchDumpPath)) { return null; } @@ -272,7 +325,41 @@ private static ReferenceChainAssertions.ChainMatch findMatch(Path scratchDumpPat try (InputStream in = Files.newInputStream(scratchDumpPath)) { events = JfrLoaderToolkit.loadEvents(in); } - return ReferenceChainAssertions.findMatchForClass( - events.apply(ItemFilters.type("datadog.ReferenceChain")), byte[].class); + IItemCollection chains = events.apply(ItemFilters.type("datadog.ReferenceChain")); + return requiredTargetTag == 0 + ? ReferenceChainAssertions.findMatchForClass(chains, byte[].class) + : ReferenceChainAssertions.findMatchForClassAndTargetTag( + chains, byte[].class, requiredTargetTag); + } + + /** + * Frontier states of LEAK_BUFFER, its backing ArrayList and that list's elementData array. + * The holders are fetched fresh on every call and never kept beyond it - a static field or a + * long-lived local would add a second root path to them. + */ + private static int[] holderStates(Field backingListField, Field elementDataField) + throws IllegalAccessException { + Object backing = backingListField.get(LEAK_BUFFER); + Object elementData = elementDataField.get(backing); + return new int[] { + JavaProfiler.getReferenceChainFrontierStateForTest0(LEAK_BUFFER), + JavaProfiler.getReferenceChainFrontierStateForTest0(backing), + JavaProfiler.getReferenceChainFrontierStateForTest0(elementData) + }; + } + + private static boolean leakBufferHoldersExpanded(Field backingListField, Field elementDataField) + throws IllegalAccessException { + for (int state : holderStates(backingListField, elementDataField)) { + if (state != FRONTIER_STATE_EXPANDED) { + return false; + } + } + return true; + } + + private static String describeHolderStates(Field backingListField, Field elementDataField) + throws IllegalAccessException { + return Arrays.toString(holderStates(backingListField, elementDataField)); } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java index b0f2e10ee2..f9894eea9a 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -108,11 +108,12 @@ byte[] take() { } /** - * Bounded wait for the leak thread's first publish. Only the - * late-thread mode needs it: there the thread is started after the - * profiler (and thus after {@code profiler.execute()} returned), so the - * main thread cannot rely on the start-command's JNI work having - * scheduled the leak thread far enough to publish. + * Bounded wait for the leak thread's first publish. Required in both + * thread-start modes: run() consumes the handoff right after the + * profiler starts, and neither starting the thread before + * {@code profiler.execute()} nor after it guarantees the thread has + * allocated its seed chunks and published by then. Must be called + * before take() (it polls seedChunk, which take() clears). */ boolean awaitPublication(long timeoutMs) throws InterruptedException { long deadline = System.nanoTime() @@ -177,17 +178,22 @@ public static void run(JavaProfiler profiler, String startCommand, // always allocated on the same qualifying tid). BlockingQueue rounds = new ArrayBlockingQueue<>(32); Thread leakThread = new Thread(() -> { - List sink = SINK.get(); + // No local ever holds the sink: re-fetch it via SINK on every use so + // the only durable path to the chunks is Thread.threadLocals -> + // ThreadLocalMap. A live stack local would let root enumeration reach + // ArrayList -> elementData -> byte[] directly (a depth-2 stack-local + // chain the transient-root gate does not suppress), bypassing the + // thread-scoped walk this scenario exists to prove. for (int i = 0; i < SEED_CHUNKS; i++) { - sink.add(new byte[CHUNK_BYTES]); + SINK.get().add(new byte[CHUNK_BYTES]); } - handoff.publish(JavaProfiler.getTid(), sink.get(0)); + handoff.publish(JavaProfiler.getTid(), SINK.get().get(0)); while (true) { try { if (rounds.poll(50, TimeUnit.MILLISECONDS) == null) { continue; } - sink.add(new byte[CHUNK_BYTES]); + SINK.get().add(new byte[CHUNK_BYTES]); } catch (InterruptedException e) { return; } @@ -204,12 +210,14 @@ public static void run(JavaProfiler profiler, String startCommand, if (lateThread) { leakThread.start(); - // The debug seeding below consumes the handoff immediately; the - // thread is only just started, so wait for its first publish. - if (!handoff.awaitPublication(30000)) { - System.out.println(NOT_FOUND_MARKER); - return; - } + } + // The debug seeding below consumes the handoff (rep chunk + tid) + // immediately. Neither thread-start mode guarantees the leak thread has + // published by now (profiler.execute() returning says nothing about the + // leak thread's progress), so wait for its first publish in both. + if (!handoff.awaitPublication(30000)) { + System.out.println(NOT_FOUND_MARKER); + return; } boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); @@ -226,6 +234,10 @@ public static void run(JavaProfiler profiler, String startCommand, JavaProfiler.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); } + // Maintenance continues the seeded ramp (next epoch = + // SEED_EPOCHS_FOR_HYSTERESIS + 1) instead of restarting at 1 - same as + // StaticFieldGrowingCollectionScenario. + hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; // Wait for the BFS thread's first full pass, then poll - same // load-bearing wait as LeakTagCorrelationScenario. int initialPasses = JavaProfiler.referenceChainPassesRunForTest0(); @@ -237,6 +249,10 @@ public static void run(JavaProfiler profiler, String startCommand, } JavaProfiler.pollReferenceChainTargets0(); } else { + // Drop the handoff's chunk reference: the lambda (captured by the + // Thread's runnable) retains `handoff`, so a never-taken seedChunk + // would be a second, non-ThreadLocalMap path to a chunk. + handoff.take(); Thread.sleep(5000); } @@ -301,6 +317,19 @@ public static void run(JavaProfiler profiler, String startCommand, System.out.println(FOUND_MARKER + best.payloadChain.targetTag); } + /** True if any hop of the leaf-to-root chain is the ThreadLocalMap (or one of its nested types). */ + private static boolean passesThroughThreadLocalMap(Object[] rawChain) { + for (Object element : rawChain) { + if (element instanceof IMCType) { + String name = ((IMCType) element).getFullName(); + if (name != null && name.contains("ThreadLocalMap")) { + return true; + } + } + } + return false; + } + private static ScanResult keepBest(ScanResult best, ScanResult scan) { if (scan == null) { return best; @@ -312,7 +341,9 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { } /** - * Scans one dump: the first leaked {@code byte[]} chain (if any), whether + * Scans one dump: the first leaked {@code byte[]} chain (if any) that runs + * through a ThreadLocalMap hop - any other byte[] chain does not prove the + * thread-scoped walk and is ignored - whether * its targetTag is in the leak-tag pool, and how many * {@code datadog.HeapLiveObject} events for a leaked {@code byte[]} carry * that same tag in {@code leakTag}. Simplified from @@ -350,6 +381,13 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { if (!"byte[]".equals(leaf)) { continue; } + // Only a chain through the leaking thread's ThreadLocalMap proves the + // thread-scoped walk; any other byte[] chain must not satisfy this + // scenario (with thread registration broken it would be the only + // kind of chain left, and the scenario must report NOT_FOUND). + if (!passesThroughThreadLocalMap(rawChain)) { + continue; + } boolean inPool = targetTag >= LEAK_TAG_BASE && targetTag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE; // Prefer the first in-pool (correlated) chain over any out-of-pool From 337acc52e7384a35869c83fe8faba607bd2f457d Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 13 Oct 2026 16:00:37 +0200 Subject: [PATCH 07/28] reference-chains: serialize frontier/resolved-tag test seams on _engine_lock frontierStateForTest() read _frontier with no lock while sibling seams take _engine_lock; resolvedChainTargetTagForTest() sampled _search_start_ns before taking _resolved_chains_lock. Take _engine_lock -> _resolved_chains_lock in both, cover the states, sentinels and lock ordering with gtests, and document the retired EDGE state and the -DDEBUG seam set. --- .../src/main/cpp/referenceChainEvents.cpp | 8 +- .../src/main/cpp/referenceChainFrontier.h | 7 +- ddprof-lib/src/main/cpp/referenceChains.cpp | 9 +- ddprof-lib/src/main/cpp/referenceChains.h | 23 ++- .../com/datadoghq/profiler/JavaProfiler.java | 18 +- .../test/cpp/referenceChainsTestAccessors.h | 8 + .../test/cpp/referenceChainsTrackerTests.inc | 167 ++++++++++++++++++ ...phinx-review-high-the-new-frontierstate.md | 82 +++++++++ 8 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 docs/sphinx/specs/2026-10-13-sphinx-review-high-the-new-frontierstate.md diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index 40244eeee2..5c2fd1e651 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -503,11 +503,17 @@ jlong ReferenceChainTracker::resolvedChainTargetTagForTest(jlong object_tag) { if (object_tag <= 0 || isLeakTag(object_tag)) { return 0; } + // _search_start_ns is only rewritten by runPass()/restartSearch() under _engine_lock (BFS thread) + // or after stopThread() - NOT under _resolved_chains_lock - so hold _engine_lock too, then sample + // the generation only once both locks are held: a concurrent restart/new pass cannot change it + // between the load and the source_search_ns comparison below. Lock order _engine_lock -> + // _resolved_chains_lock matches finishLoopIterationSerialized() -> pollWatchedTargets(). + MutexLocker engine_guard(_engine_lock); + ExclusiveLockGuard guard(&_resolved_chains_lock); const u64 current_search_ns = load(_search_start_ns); if (current_search_ns == 0) { return 0; } - ExclusiveLockGuard guard(&_resolved_chains_lock); auto it = _resolved_chains.find(object_tag); if (it == _resolved_chains.end() || it->second.source_search_ns != current_search_ns) { diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.h b/ddprof-lib/src/main/cpp/referenceChainFrontier.h index 093ed11ffc..9228b516e6 100644 --- a/ddprof-lib/src/main/cpp/referenceChainFrontier.h +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.h @@ -21,7 +21,9 @@ namespace FrontierEntryState { constexpr u8 FRONTIER = 0; // discovered, not yet expanded by FollowReferences constexpr u8 EXPANDED = 1; // expanded; children (if any) are in the table -constexpr u8 EDGE = 2; // on a path toward a target sample (EdgeStore) +// EDGE is retired: never set by the production walk (see reconstructChain()'s comment in +// referenceChainFrontier.cpp); only markEdge() and unit/fuzz fixtures set it. +constexpr u8 EDGE = 2; constexpr u8 ABANDONED = 3; // tag released; entry kept only to avoid reuse } // namespace FrontierEntryState @@ -183,7 +185,8 @@ class alignas(alignof(SpinLock)) FrontierTable { // Marks metadata abandoned; the caller must clear the JVMTI tag. void clear(jlong tag); - // Marks the slot as part of a resolved path. + // Marks the slot EDGE. Retired: no production caller (only the fuzz harness's shadow model and + // unit-test fixtures use EDGE). void markEdge(jlong tag); // Marks the slot after all outgoing edges have been visited. diff --git a/ddprof-lib/src/main/cpp/referenceChains.cpp b/ddprof-lib/src/main/cpp/referenceChains.cpp index 6115831e19..211686301e 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.cpp +++ b/ddprof-lib/src/main/cpp/referenceChains.cpp @@ -1064,7 +1064,14 @@ long ReferenceChainTracker::pendingExpandPositionForTest(jlong tag) const { } int ReferenceChainTracker::frontierStateForTest(jlong object_tag) { - if (object_tag <= 0 || isLeakTag(object_tag) || _frontier == nullptr) { + if (object_tag <= 0 || isLeakTag(object_tag)) { + return -1; + } + // Same lock as the pendingExpand*ForTest() seams above: the BFS thread mutates the frontier + // (insert/expand/abandon in runPass(), resetForRestart() via shouldRunPass()'s restart branch) + // only under _engine_lock, so this snapshot never observes a pass or restart mid-flight. + MutexLocker engine_guard(_engine_lock); + if (_frontier == nullptr) { return -1; } FrontierEntry entry{}; diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index c286e2c22c..09aaecbbe3 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -1204,9 +1204,11 @@ class ReferenceChainTracker { // Held by runPassSerialized() and finishLoopIterationSerialized() around the engine work, by // threadLoop() around shouldRunPass() (whose restart branch mutates _pending_expand/ - // _priority_expand the same way a pass does), and by the pendingExpand*ForTest() seams below so - // they never observe those deques mid-mutation from the BFS thread. Mutable because the test - // seams are const (read-only from the caller's perspective) but still need to take the lock. + // _priority_expand the same way a pass does), and by the pendingExpand*ForTest()/ + // frontierStateForTest()/resolvedChainTargetTagForTest() test seams below so they never observe + // engine state mid-mutation from the BFS thread. Mutable because the test seams are const + // (read-only from the caller's perspective) but still need to take the lock. Recursive + // (mutex.cpp); never acquire it while holding _resolved_chains_lock. mutable Mutex _engine_lock; jlong tagObject(jvmtiEnv *jvmti, jobject obj); jlong getTag(jvmtiEnv *jvmti, jobject obj); @@ -1368,11 +1370,13 @@ class ReferenceChainTracker { // for computing a position's fraction of the current backlog. size_t pendingExpandSizeForTest() const; - // Test seam - not part of the production API. The FrontierEntryState of the current search's - // frontier entry for the object carrying JVMTI tag `object_tag` (FRONTIER/EXPANDED/ABANDONED), - // or -1 if there is no such entry: untagged, leak-tagged (not yet intercepted by the walk), or - // no frontier table / slot. Lets a test synchronize on an object's actual expansion rather than - // inferring it from pass counters. + // Test seam - not part of the production API. The raw FrontierEntryState (canonical values: + // referenceChainFrontier.h) of the current search's frontier entry for the object carrying JVMTI + // tag `object_tag` - FRONTIER(0)/EXPANDED(1)/ABANDONED(3); EDGE(2) is retired and never produced + // by the production walk (FrontierTable::markEdge() has no production caller) - or -1 if there is + // no such entry: untagged, leak-tagged (not yet intercepted by the walk), or no frontier table / + // slot. Lets a test synchronize on an object's actual expansion rather than inferring it from + // pass counters. Serialized with the BFS thread via _engine_lock. int frontierStateForTest(jlong object_tag); // Test seam - not part of the production API. Returns the targetTag a datadog.ReferenceChain @@ -1381,7 +1385,8 @@ class ReferenceChainTracker { // keyed by its frontier tag with source_search_ns == _search_start_ns). 0 otherwise: untagged, // leak-tagged but not yet intercepted, no cached chain, or a chain cached by an earlier search // generation (restartSearch() keeps _resolved_chains and reuses tags from 1, so raw tag equality - // alone could match a stale, unrelated chain). + // alone could match a stale, unrelated chain). Samples _search_start_ns under _engine_lock and + // _resolved_chains_lock. jlong resolvedChainTargetTagForTest(jlong object_tag); // Test seam - not part of the production API. Evaluates the real private scheduling gate without diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index a876264a24..5c341214a6 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -672,17 +672,21 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l public static native long getReferenceChainPendingSizeForTest0(); /** - * Test seam (debug native builds only, returns -1 in release): does not tag {@code target}. - * The state of the current search's frontier entry for target - 0 = FRONTIER (admitted, not - * yet expanded), 1 = EXPANDED, 3 = ABANDONED - or {@code -1} if target has no frontier entry - * (never admitted, or still carrying only a leak tag). + * Test seam (-DDEBUG native builds only - debug/asan/tsan; returns -1 in release): does not + * tag {@code target}. The raw {@code FrontierEntryState} (canonical values: + * referenceChainFrontier.h) of the current search's frontier entry for target - 0 = FRONTIER + * (admitted, not yet expanded), 1 = EXPANDED, 2 = EDGE (retired: never produced by the + * production walk, so this seam does not return it in practice), 3 = ABANDONED - or + * {@code -1} if target has no frontier entry (never admitted, or still carrying only a leak + * tag). */ public static native int getReferenceChainFrontierStateForTest0(Object target); /** - * Test seam (debug native builds only, returns 0 in release): does not tag {@code target}. - * The targetTag the {@code datadog.ReferenceChain} event for target carries, if the current - * search has already resolved and cached target's own chain; 0 otherwise. + * Test seam (-DDEBUG native builds only - debug/asan/tsan; returns 0 in release): does not + * tag {@code target}. The targetTag the {@code datadog.ReferenceChain} event for target + * carries, if the current search has already resolved and cached target's own chain; 0 + * otherwise. */ public static native long getReferenceChainResolvedTargetTagForTest0(Object target); diff --git a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h index ef2a0d6817..22d345521e 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h +++ b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h @@ -217,6 +217,14 @@ class ReferenceChainsTestAccessor { static size_t resolvedChainCount() { return ReferenceChainTracker::instance()->_resolved_chains.size(); } + // Holds/releases the private _resolved_chains_lock, so a test can prove a seam takes + // _engine_lock BEFORE _resolved_chains_lock (and never the reverse). + static void lockResolvedChainsForTest() { + ReferenceChainTracker::instance()->_resolved_chains_lock.lock(); + } + static void unlockResolvedChainsForTest() { + ReferenceChainTracker::instance()->_resolved_chains_lock.unlock(); + } // Source tags of every resolved chain - the fuzz target iterates these to // check for phantom/negative-tag chains (I2's sign discrimination). static std::vector resolvedChainSourceTags() { diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc index 3fdecb8a36..3f9124d7d6 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -676,6 +676,173 @@ TEST_F(ResolvedChainCacheTest, FullCacheEvictsPreviousGenerationForCurrentChain) EXPECT_EQ((u64)kNewTag, dump[0]._target_tag); } +// Test-seam locking (frontierStateForTest()/resolvedChainTargetTagForTest()): both seams are +// called from a Java test thread while the BFS thread mutates the frontier / search generation +// under _engine_lock, so they must serialize on that same lock. _engine_lock is recursive, so the +// blocking is only observable from a second thread. +namespace { +// Polls `done` for up to `timeout_ms`; true once it is set. +bool waitForSeamThread(const std::atomic &done, int timeout_ms) { + for (int waited = 0; waited < timeout_ms; waited += 10) { + if (done.load()) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return done.load(); +} +} // namespace + +// Every value the frontier-state seam can return, and every -1 sentinel path. +TEST_F(ReferenceChainsBfsTest, FrontierStateSeamReportsStatesAndSentinels) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + ASSERT_NE(nullptr, frontier); + + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::FRONTIER, 0)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 2, 0, 0, FrontierEntryState::EXPANDED, 0)); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 3, 0, 0, FrontierEntryState::ABANDONED, 0)); + + EXPECT_EQ(0, tracker->frontierStateForTest(1)); + EXPECT_EQ(1, tracker->frontierStateForTest(2)); + EXPECT_EQ(3, tracker->frontierStateForTest(3)); + + EXPECT_EQ(-1, tracker->frontierStateForTest(0)) << "untagged"; + EXPECT_EQ(-1, tracker->frontierStateForTest(-7)) << "negative tag"; + EXPECT_EQ(-1, tracker->frontierStateForTest(999)) << "no frontier entry"; + EXPECT_EQ(-1, tracker->frontierStateForTest(ReferenceChainTracker::LEAK_TAG_BASE)) + << "leak tag"; + EXPECT_EQ(-1, tracker->frontierStateForTest(ReferenceChainTracker::LEAK_TAG_BASE + 5)) + << "leak tag"; + + tracker->stop(); + // No frontier table at all: -1, not a crash. + ReferenceChainsTestAccessor::reset(); + EXPECT_EQ(-1, tracker->frontierStateForTest(1)); +} + +// frontierStateForTest() must wait for a holder of _engine_lock (a running pass) and report the +// state that holder left behind - while the pure tag guard stays a non-blocking fast path. +TEST_F(ReferenceChainsBfsTest, FrontierStateSeamSerializesWithEngineLock) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + ASSERT_NE(nullptr, frontier); + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 1, 0, 0, FrontierEntryState::FRONTIER, 0)); + + tracker->_engine_lock.lock(); + std::atomic done{false}; + std::atomic result{-2}; + std::thread seam([&] { + result.store(tracker->frontierStateForTest(1)); + done.store(true); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_FALSE(done.load()) << "seam must block while _engine_lock is held"; + + // Fast path: the tag guard never touches engine state, so it must not block. + std::atomic fast_done{false}; + std::atomic fast_result{-2}; + std::thread fast([&] { + fast_result.store(tracker->frontierStateForTest(0)); + fast_done.store(true); + }); + EXPECT_TRUE(waitForSeamThread(fast_done, 2000)) << "tag guard must not take _engine_lock"; + + // Simulates a pass expanding the entry while it holds _engine_lock. + frontier->markExpanded(1); + tracker->_engine_lock.unlock(); + + EXPECT_TRUE(waitForSeamThread(done, 10000)); + seam.join(); + fast.join(); + EXPECT_EQ(1, result.load()); + EXPECT_EQ(-1, fast_result.load()); + + tracker->stop(); +} + +// The search-generation guard of resolvedChainTargetTagForTest(): every 0 path and the match. +TEST_F(ResolvedChainCacheTest, ResolvedTargetTagSeamChecksGeneration) { + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeEvent(77), 5, 1000); + EXPECT_EQ(77, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(5)); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(6)) + << "no cached chain"; + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(0)); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(-1)); + + const jlong leakTag = ReferenceChainTracker::LEAK_TAG_BASE; + ReferenceChainsTestAccessor::cacheChain(leakTag, makeEvent(88), leakTag, 1000); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(leakTag)) + << "leak tags are never resolved-chain keys for this seam"; + + ReferenceChainsTestAccessor::setSearchStartNsForTest(2000); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(5)) + << "chain cached by an earlier search generation"; + + ReferenceChainsTestAccessor::setSearchStartNsForTest(0); + ReferenceChainsTestAccessor::cacheChain(7, makeEvent(99), 7, 0); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(7)) + << "no search in flight: 0 even though source_search_ns == 0 matches"; +} + +// A generation change made under _engine_lock (restartSearch()/runPass()) while the seam waits must +// be observed: the seam samples _search_start_ns only once it holds _engine_lock. +TEST_F(ResolvedChainCacheTest, ResolvedTargetTagSeamSamplesGenerationUnderEngineLock) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeEvent(77), 5, 1000); + + tracker->_engine_lock.lock(); + std::atomic done{false}; + std::atomic result{-2}; + std::thread seam([&] { + result.store(tracker->resolvedChainTargetTagForTest(5)); + done.store(true); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_FALSE(done.load()) << "seam must block while _engine_lock is held"; + ReferenceChainsTestAccessor::setSearchStartNsForTest(2000); + tracker->_engine_lock.unlock(); + + EXPECT_TRUE(waitForSeamThread(done, 10000)); + seam.join(); + EXPECT_EQ(0, result.load()) << "stale-generation chain returned after a concurrent restart"; +} + +// Lock order _engine_lock -> _resolved_chains_lock: with only _resolved_chains_lock held, the seam +// takes _engine_lock, waits on the cache lock, and completes once it is released. +TEST_F(ResolvedChainCacheTest, ResolvedTargetTagSeamLockOrder) { + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeEvent(77), 5, 1000); + + ReferenceChainsTestAccessor::lockResolvedChainsForTest(); + std::atomic done{false}; + std::atomic result{-2}; + std::thread seam([&] { + result.store(tracker->resolvedChainTargetTagForTest(5)); + done.store(true); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_FALSE(done.load()) << "seam must wait on _resolved_chains_lock"; + ReferenceChainsTestAccessor::unlockResolvedChainsForTest(); + + EXPECT_TRUE(waitForSeamThread(done, 10000)); + seam.join(); + EXPECT_EQ(77, result.load()); +} + // Pause-time pacing controller: pause-time-SLO feedback loop TEST_F(ReferenceChainsTest, PacingHoldsSteadyWhenPassesLandExactlyOnCeiling) { diff --git a/docs/sphinx/specs/2026-10-13-sphinx-review-high-the-new-frontierstate.md b/docs/sphinx/specs/2026-10-13-sphinx-review-high-the-new-frontierstate.md new file mode 100644 index 0000000000..03aded3da0 --- /dev/null +++ b/docs/sphinx/specs/2026-10-13-sphinx-review-high-the-new-frontierstate.md @@ -0,0 +1,82 @@ +# Spec: Reference-chain test seams — locking, contract docs, scenario predicate and JDK-8 args + +This spec covers PR-800 change requests F-91f3cc84a25e, F-17cff193f8ed, F-054807ffb76e, F-3b9270c81830, F-b2e5089803c3 and F-32af0fcc0ab2. + +The design is in `.sphinx/address/design.md`. Its "Cluster Rules & Invariants" section is the normative source of truth for this spec. + +## Problem + +The PR added two native test seams. Java test scenarios use them to stay in step with the reference-chain BFS engine. + +- `ReferenceChainTracker::frontierStateForTest()` reads `_frontier` without `_engine_lock`. Every other engine-state seam takes that lock, so this seam can see a pass or a restart while it is only partly done (F-91f3cc84a25e). Its guard and sentinel paths have no tests (F-32af0fcc0ab2). +- `ReferenceChainTracker::resolvedChainTargetTagForTest()` samples the search generation (`_search_start_ns`) before it takes any lock. A concurrent `restartSearch()` or new `runPass()` can then change the generation between the sample and the comparison against the cached entry. The seam can return a stale chain's target tag (F-17cff193f8ed). + - The real writers of the generation hold `_engine_lock`, not `_resolved_chains_lock`. Reordering the load relative to `_resolved_chains_lock` alone does not close the window. +- The javadoc and header comment for the frontier-state seam skip state value 2 (EDGE) without explaining why (F-3b9270c81830). +- `StaticFieldGrowingCollectionScenario` enables its seam-based synchronization only when the config is `"debug"`. The seams are also compiled into asan and tsan builds, so the strongest race-prevention path never runs on sanitizer CI (F-b2e5089803c3). +- `ExternalProcessReferenceChainTest` always passes `--add-opens`. A JDK 8 child JVM rejects that option (F-054807ffb76e). + +## Correct behaviour + +1. **R-LOCK (seam locking).** + - `frontierStateForTest(tag)` and `resolvedChainTargetTagForTest(tag)` hold `_engine_lock` for their entire read of engine state. This matches `pendingExpandPositionForTest()` and `pendingExpandSizeForTest()`. + - If another thread holds `_engine_lock`, a seam call with a valid tag does not return until that thread releases it. + - The invalid-tag fast path returns the sentinel without locking. Invalid means `tag <= 0`, or a leak tag. +2. **R-ORDER (lock order).** + - The lock order is `_engine_lock` first, then `_resolved_chains_lock`. This is the same order as the BFS thread (`finishLoopIterationSerialized` → `pollWatchedTargets`, and `cacheResolvedChain`). + - No path holds `_resolved_chains_lock` while it waits on `_engine_lock`. + - A seam call that runs concurrently with a lock holder finishes once that holder releases. It never deadlocks. +3. **R-GEN (generation guard).** `resolvedChainTargetTagForTest(tag)` returns the cached `_target_tag` only when all of the following hold. Otherwise it returns 0. + - `tag > 0`. + - The tag is not a leak tag. + - `_search_start_ns != 0`. + - `cached.source_search_ns == _search_start_ns`, with `_search_start_ns` read while both locks are held. + + If a lock holder changes the generation while a seam call is waiting, the seam call sees the new generation. +4. **R-SENTINEL (frontier-state contract).** `frontierStateForTest(tag)` returns -1 in each of these cases: + - `tag <= 0`. + - `tag` is a leak tag. + - There is no frontier table. + - There is no entry for `tag`. + + Otherwise it returns the raw `FrontierEntryState`: FRONTIER = 0, EXPANDED = 1, ABANDONED = 3. EDGE = 2 is retired; the production walk never produces it. From Java, an object that has never been tagged reads as -1. +5. **R-BUILD (scenario seam-path predicate).** `StaticFieldGrowingCollectionScenario` takes the seam path (holder-EXPANDED wait, lateChunk identity pinning, untagged-probe check) exactly when `config != null && ("debug".equals(config) || config.endsWith("san"))`. Release builds take the class-only path. The 45 s holder deadline and the 150 s parent launch timeout stay as they are. +6. **R-JDK (child JVM args).** The `leak-static-field` child always receives `-Dddprof_test.config=`. It receives `--add-opens=java.base/java.util=ALL-UNNAMED` if and only if `Platform.isJavaVersionAtLeast(9)`. +7. **R-DOC (single source of truth).** + - `FrontierEntryState` in `referenceChainFrontier.h` is the canonical numbering. + - The JavaProfiler javadoc and the `referenceChains.h` seam comment list all four values. Both say that 2 = EDGE is retired. + - The `EDGE` and `markEdge` comments say they have no production caller. + - No numeric values change. + +## Constraints + +- No production behaviour change. Every native change touches test-only seams or comments. +- Keep the seam signatures, the JNI shims (`javaApi.cpp`), the release stubs, and the `FrontierEntryState` numeric values unchanged. Keep `markEdge`, because the fuzz harness uses it. +- `Mutex` is recursive (`PTHREAD_MUTEX_RECURSIVE`). Never take `_engine_lock` while holding `_resolved_chains_lock`. +- Java formatting goes through spotless. C++ style follows the existing two-space, ~100-column comments. +- Profiler hot paths are not touched. Both seams are called only from Java test threads via JNI. + +## Scope + +### Primary fixes + +- `ddprof-lib/src/main/cpp/referenceChains.cpp:1066` (F-91f3cc84a25e). Defect class: a test seam reads engine state without `_engine_lock`. Fix: take `_engine_lock` after the tag guard, and move the `_frontier == nullptr` check under the lock. +- `ddprof-lib/src/main/cpp/referenceChainEvents.cpp:502` (F-17cff193f8ed). Defect class: a generation stamp is sampled outside the lock that serializes its writers. Fix: take `_engine_lock`, then `_resolved_chains_lock`, then load `_search_start_ns`. +- `ddprof-test/.../ExternalProcessReferenceChainTest.java:174` (F-054807ffb76e). Defect class: a JDK 9+ only JVM option is passed to the child unconditionally. Fix: add `--add-opens` only when `Platform.isJavaVersionAtLeast(9)`. +- `ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java:677` (F-3b9270c81830). Defect class: an incomplete enum contract in the docs. Fix: document all four states in the javadoc, in `referenceChains.h`, and in the `referenceChainFrontier.h` EDGE and `markEdge` comments. +- `ddprof-test/.../StaticFieldGrowingCollectionScenario.java:100` (F-b2e5089803c3). Defect class: the seam-path predicate is narrower than the builds where the seams exist. Fix: change it to debug-or-`*san`. +- `ddprof-lib/src/main/cpp/referenceChains.cpp:1067` (F-32af0fcc0ab2). Defect class: the sentinel and lock paths have no tests. Fix: the lock is covered by the first fix; add the untagged-probe `-1` assertion in the scenario before seeding. Native gtests per the tester plan. + +### Auto-expanded sibling fixes + +None identified. The candidates below were considered and deferred; see `planner-report.json`'s `architecture_notes`. + +- **Debug-only predicate in other scenarios** (`ThreadLocalLeakScenario:223`, `LeakTagCorrelationScenario:244`, `LeakingCacheScenario:131`, and others). There the predicate only gates a seeding shortcut, not an identity or race guarantee. Widening it would change those tests' behaviour on sanitizer builds, and there is no runtime evidence that would hold up. The review did not ask for it. +- **`tagAsRootForTest`**. It is pre-existing, it mutates engine state rather than reading it (a different defect class), and this PR does not touch it. + +## Assumptions + +- I am at least 90% confident that taking `_engine_lock` in `resolvedChainTargetTagForTest()` is the minimal change that actually satisfies the reviewer's intent. The full writer list was grep-verified. +- A Java test thread blocked in native code on `_engine_lock` for up to one pass does not stall safepoints. The `pendingExpand*` seams already rely on the same pattern. +- `Platform.isJavaVersionAtLeast(9)` reflects the test JVM, which in the gradle matrix is the same JDK the child runs on. +- **Needs runtime evidence:** whether the strict path on asan/tsan completes within 45 s. +- **Residual, out of scope:** the JNI shims call `GetTag` outside `_engine_lock`, which is a time-of-check/time-of-use gap. From 849b5603cbae6a1af4144a2f0767f1f21ec31ad0 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 13 Oct 2026 16:00:37 +0200 Subject: [PATCH 08/28] reference-chains tests: gate seam path on -DDEBUG builds, guard --add-opens on JDK 8 The scenario's native seams compile into debug/asan/tsan, so enable the seam path on all three (asserted via a stdout marker) instead of config==debug only. --add-opens is rejected by JDK 8's launcher, so add it only on JDK 9+. --- .../ExternalProcessReferenceChainTest.java | 21 ++++++-- .../StaticFieldGrowingCollectionScenario.java | 49 ++++++++++++++----- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java index 8e098e757f..a35b61913a 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -12,7 +12,7 @@ import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -167,13 +167,17 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); + List jvmArgs = new ArrayList<>(); + jvmArgs.add("-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); // --add-opens: the scenario reflects into Collections$SynchronizedList.list and // ArrayList.elementData to check its holders' expansion state before creating lateChunk. - List jvmArgs = Arrays.asList( - "-Dddprof_test.config=" + System.getProperty("ddprof_test.config"), - "--add-opens=java.base/java.util=ALL-UNNAMED"); + // JDK 8 has no module system (that reflection needs no opening) and rejects the option. + if (Platform.isJavaVersionAtLeast(9)) { + jvmArgs.add("--add-opens=java.base/java.util=ALL-UNNAMED"); + } AtomicReference resultLine = new AtomicReference<>(); + AtomicReference seamPathLine = new AtomicReference<>(); LaunchResult result = launch("leak-static-field", jvmArgs, packedCommand, Collections.emptyMap(), // Wider than shouldReconstructReferrerChainInSeparateProcess()'s 90s: this scenario's @@ -185,6 +189,8 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce if (line.startsWith(StaticFieldGrowingCollectionScenario.FOUND_MARKER) || line.equals(StaticFieldGrowingCollectionScenario.NOT_FOUND_MARKER)) { resultLine.set(line); + } else if (line.startsWith(StaticFieldGrowingCollectionScenario.SEAM_PATH_MARKER)) { + seamPathLine.set(line); } return LineConsumerResult.CONTINUE; }, @@ -192,6 +198,13 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce assertTrue(result.inTime, "Child process did not exit within the wait timeout"); assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); + // The scenario's native test seams are compiled into every -DDEBUG build (debug/asan/tsan), + // so the strong seam path must be taken on exactly those configs. + String config = System.getProperty("ddprof_test.config"); + boolean expectSeams = config != null && ("debug".equals(config) || config.endsWith("san")); + assertEquals(StaticFieldGrowingCollectionScenario.SEAM_PATH_MARKER + expectSeams, + seamPathLine.get(), + "scenario's seam-path predicate must match the native -DDEBUG build set"); assertNotNull(resultLine.get(), "Child process never printed a recognizable result marker on stdout"); assertEquals( diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java index 6f284d9f17..4b525c4441 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java @@ -76,9 +76,18 @@ private StaticFieldGrowingCollectionScenario() {} // JavaProfiler.getReferenceChainFrontierStateForTest0(). private static final int FRONTIER_STATE_EXPANDED = 1; + // JavaProfiler.getReferenceChainFrontierStateForTest0()'s "no frontier entry" sentinel. + private static final int FRONTIER_STATE_NONE = -1; + /** Printed to stdout, followed by the matched leaf class's name, on success. */ public static final String FOUND_MARKER = "[chain-found] "; + /** + * Printed to stdout, followed by {@code true}/{@code false}, once the scenario has decided + * whether it takes the native test-seam path (every -DDEBUG build: debug/asan/tsan). + */ + public static final String SEAM_PATH_MARKER = "[seam-path] "; + /** Printed to stdout (with no class name suffix) if no match was ever observed. */ public static final String NOT_FOUND_MARKER = "[chain-not-found]"; @@ -97,7 +106,14 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD profiler.execute(startCommand); } - boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + // The seams used below are compiled into every -DDEBUG native build - debug, asan and tsan + // (javaApi.cpp's #ifdef DEBUG block; ConfigurationPresets.kt) - so the seam path must be + // enabled on all three. Same predicate as LeakTagCorrelationReferenceChainTest's rotation + // gate; release builds get the no-op stubs. + String testConfig = System.getProperty("ddprof_test.config"); + boolean debugBuild = testConfig != null + && ("debug".equals(testConfig) || testConfig.endsWith("san")); + System.out.println(SEAM_PATH_MARKER + debugBuild); int hysteresisEpoch = 0; // Declared at method scope: the per-round maintenance seeding below (in // its own debugBuild block) reuses this tid. @@ -108,6 +124,14 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD int initialPasses = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; if (debugBuild) { + // Pins the frontier-state seam's -1 sentinel: an untagged object (tag 0) has no frontier + // entry. Probed before any seeding - no pass can run until shouldRunPass() sees a leak + // signal - so the BFS root walk cannot have tagged the probe yet. + int untaggedState = JavaProfiler.getReferenceChainFrontierStateForTest0(new Object()); + if (untaggedState != FRONTIER_STATE_NONE) { + throw new IllegalStateException("frontier-state seam returned " + untaggedState + + " for an untagged probe object, expected " + FRONTIER_STATE_NONE); + } // Seed enough hysteresis-qualifying epochs to unlock ReferenceChainTracker::hasLeakSignal()/ // shouldRunPass() - WITHOUT wiring in a representative object yet (setKlassPopulationRepresentativeForTest0 // is not called here). hasLeakSignal() only asks selectLeakCandidates() for a nonzero count - @@ -136,11 +160,11 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // admitStaticFieldRoots() and expandFrontier() are two steps of the SAME runPass() call, and // neither the static-field sweep's completion nor a changed pass counter proves the holders // were expanded (see the debug branch below), so the wait reads their frontier state directly. - // Non-debug builds have no such seam and fall back to a generous sleep - acceptable there - // because the non-debug path never pins a specific representative anyway (see the debugBuild - // branch below): it just waits on real, much-slower allocation sampling to notice the growth - // on its own, so a few extra seconds of margin either way changes nothing about what the - // eventual match would prove. + // Release (non-DEBUG) builds have no such seam and fall back to a generous sleep - acceptable + // there because the non-debug path never pins a specific representative anyway (see the + // debugBuild branch below): it just waits on real, much-slower allocation sampling to notice + // the growth on its own, so a few extra seconds of margin either way changes nothing about + // what the eventual match would prove. if (debugBuild) { // A changed pass counter alone is not proof that LEAK_BUFFER's holders were expanded: // truncated passes also bump it, and admitStaticFieldRoots() only processes a bounded @@ -167,7 +191,9 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // against pause-time-SLO pacing, and the expansion backlog ahead of LEAK_BUFFER's holders // is FIFO - generous margin for the same reason ExternalProcessReferenceChainTest's own // budget/pausetarget are raised this far above the in-process test's defaults. Wall-clock - // deadline: the seam reads can contend with a running pass's locks. + // deadline: the seam reads can contend with a running pass's locks. Not scaled for + // asan/tsan: the parent's 150s launch timeout must still cover this wait + rounds + the 60s + // tail. long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(45); boolean holdersExpanded = false; while (!holdersExpanded && System.nanoTime() < deadline) { @@ -266,7 +292,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } } - // Non-debug builds have no identity seam: class-only matching, as before. + // Release (non-DEBUG) builds have no identity seam: class-only matching, as before. if (match == null && !debugBuild) { profiler.dump(scratchDumpPath); match = findMatch(scratchDumpPath, 0); @@ -287,9 +313,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // far for exactly this reason. for (int attempt = 0; match == null && attempt < 40; attempt++) { Thread.sleep(1500); - // Same passesRun()-advanced check as the round loop above (debug builds only) - by this - // point in a real run it has virtually always already flipped, but there is no reason to - // drop the guarantee here just because the round loop above didn't need it. + // Same passesRun()-advanced check as the round loop above (-DDEBUG builds only: + // debug/asan/tsan) - by this point in a real run it has virtually always already flipped, + // but there is no reason to drop the guarantee here just because the round loop above + // didn't need it. // Same lateChunk-identity pinning as the round loop above. long requiredTargetTag = 0; if (debugBuild) { From a66dbc987d54566462fec57099611bf4effabc8e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Wed, 14 Oct 2026 13:52:55 +0200 Subject: [PATCH 09/28] sphinx: address review feedback on PR #800 --- ddprof-lib/src/main/cpp/javaApi.cpp | 27 +++++++--- .../LeakTagCorrelationScenario.java | 31 +++++++++--- .../ReferenceChainTrackingTest.java | 49 ++++++++++++------- 3 files changed, 76 insertions(+), 31 deletions(-) diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 36b55c5370..89db648616 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1263,6 +1263,13 @@ Java_com_datadoghq_profiler_JavaProfiler_resetReferenceChainSearchForTest0( // Reads target's existing tag without tagging it. Return values are described // at pendingExpandPositionForTest(). +// +// The tag read and the lookup both run under _engine_lock: the BFS thread +// releases and reissues frontier tags (releaseSearchTags()/restartSearch()) +// only while holding it, so the tag cannot be reassigned to another object +// between GetTag() and the lookup. The lock is recursive (mutex.cpp), so the +// tracker seam's own nested acquisition is fine. The same applies to the two +// seams below. extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingPositionForTest0( JNIEnv *env, jclass unused, jobject target) { @@ -1271,13 +1278,14 @@ Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingPositionForTest if (jvmti == nullptr || target == nullptr) { return -2; } + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + MutexLocker engine_guard(tracker->_engine_lock); jlong tag = 0; jvmtiError err = jvmti->GetTag(target, &tag); if (err != JVMTI_ERROR_NONE) { return -2; } - return (jlong)ReferenceChainTracker::instance()->pendingExpandPositionForTest( - tag); + return (jlong)tracker->pendingExpandPositionForTest(tag); } extern "C" DLLEXPORT jlong JNICALL @@ -1287,7 +1295,8 @@ Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainPendingSizeForTest0( return (jlong)ReferenceChainTracker::instance()->pendingExpandSizeForTest(); } -// Reads target's existing tag without tagging it; see frontierStateForTest(). +// Reads target's existing tag without tagging it, under _engine_lock (see +// getReferenceChainPendingPositionForTest0()); see frontierStateForTest(). extern "C" DLLEXPORT jint JNICALL Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainFrontierStateForTest0( JNIEnv *env, jclass unused, jobject target) { @@ -1296,14 +1305,18 @@ Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainFrontierStateForTest0( if (jvmti == nullptr || target == nullptr) { return -1; } + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + MutexLocker engine_guard(tracker->_engine_lock); jlong tag = 0; if (jvmti->GetTag(target, &tag) != JVMTI_ERROR_NONE) { return -1; } - return (jint)ReferenceChainTracker::instance()->frontierStateForTest(tag); + return (jint)tracker->frontierStateForTest(tag); } -// Reads target's existing tag without tagging it; see resolvedChainTargetTagForTest(). +// Reads target's existing tag without tagging it, under _engine_lock (see +// getReferenceChainPendingPositionForTest0()); see +// resolvedChainTargetTagForTest(). extern "C" DLLEXPORT jlong JNICALL Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainResolvedTargetTagForTest0( JNIEnv *env, jclass unused, jobject target) { @@ -1312,11 +1325,13 @@ Java_com_datadoghq_profiler_JavaProfiler_getReferenceChainResolvedTargetTagForTe if (jvmti == nullptr || target == nullptr) { return 0; } + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + MutexLocker engine_guard(tracker->_engine_lock); jlong tag = 0; if (jvmti->GetTag(target, &tag) != JVMTI_ERROR_NONE) { return 0; } - return ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(tag); + return tracker->resolvedChainTargetTagForTest(tag); } // Seeds one heap-floor sample for secondsToOOM() without waiting for a GC. diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index 7b3b3b3e14..93a0c3071d 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -231,7 +231,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } // Churn one slot at a time: the population stays live-but-ephemeral (constant live // count, nothing durably retained beyond the frame) while still allocating. - held[(int) (slot++ % held.length)] = new NoisePayload(); + // held[0] is reserved: it is the published noise representative, and after + // handoff.take() this frame is its only strong reference (the tracker holds just a + // weak ref) - churning it would let the representative be collected. + held[1 + (int) (slot++ % (held.length - 1))] = new NoisePayload(); } }, "leak-correlation-noise"); noiseThread.setDaemon(true); @@ -258,6 +261,17 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // process's life; the noise representative only via the noise thread's frame (the // handoff's own reference is dropped by take()) - exactly what makes one a valid // leak candidate and the other a gate test. + // The noise thread publishes asynchronously: wait for it, or take() below could + // return null (no alias to the real NoisePayload klass) and handoff.tid could be 0 + // (no real allocating tid qualified) - noise suppression would then go unexercised. + // Plain volatile read, NOT take(): take() drops the handoff's reference. + long noisePublicationDeadline = System.nanoTime() + 30_000_000_000L; + while (handoff.representative == null && System.nanoTime() < noisePublicationDeadline) { + Thread.sleep(10); + } + if (handoff.representative == null || handoff.tid <= 0) { + throw new IllegalStateException("Noise thread did not publish a representative and tid"); + } JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, LEAK_SINK.get(0)); JavaProfiler.setKlassPopulationRepresentativeForTest0(NOISE_KLASS_ID, handoff.take()); // Per-(klass, tid) qualification seeds: selectLeakCandidates() now also @@ -363,18 +377,21 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD best = keepBest(best, scan(scratchDumpPath)); } - if (best == null || best.payloadChain == null) { - System.out.println(NOT_FOUND_MARKER); - return; - } - if (!best.transientViolations.isEmpty()) { + // Gate violations are 'ever emitted' contracts, independent of whether a payload chain + // was found - report them first so a NOT_FOUND result (accepted by the tight-window + // variant) can never mask a suppression regression. + if (best != null && !best.transientViolations.isEmpty()) { System.out.println(TRANSIENT_CHAIN_MARKER + best.transientViolations); return; } - if (!best.noiseChains.isEmpty()) { + if (best != null && !best.noiseChains.isEmpty()) { System.out.println(NOISE_CHAIN_MARKER + best.noiseChains); return; } + if (best == null || best.payloadChain == null) { + System.out.println(NOT_FOUND_MARKER); + return; + } if (!best.payloadChainInPool) { System.out.println(TAG_OUT_OF_POOL_MARKER + best.payloadChain.targetTag); return; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java index c3fc86ec6b..9506c2b9d6 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java @@ -142,6 +142,11 @@ public class ReferenceChainTrackingTest extends AbstractProfilerTest { // for the same cheap-insurance-against-collision reason those two are distinct from each other. private static final int ABANDON_TEST_KLASS_ID = 987203; + // T_REFERENCE_CHAIN in ddprof-lib/src/main/cpp/jfrMetadata.h - the jdk.ActiveSetting "id" + // the profiler writes for the datadog.ReferenceChain setting (flightRecorder.cpp's + // writeBoolSetting(buf, T_REFERENCE_CHAIN, "enabled", ...)). Must stay in sync with that enum. + private static final long REFERENCE_CHAIN_EVENT_ID = 129; + @Override protected String getProfilerCommand() { String testName = testInfo != null @@ -257,15 +262,19 @@ protected String getProfilerCommand() { // draining that debt back to canStartNow()==true would take on the order of minutes, far past // this method's own retry budget below. painbudget=100 keeps that gate affordable immediately, // the same way it does for the sibling method. + // + // memory=64:l:1.0 is load-bearing: Profiler::start() (profiler.cpp) only starts the + // ReferenceChainTracker when allocation sampling actually activated (activated & EM_ALLOC). + // A bare referencechains=... command falls back to cpu-only profiling, so the search never + // runs and dump() could never emit datadog.ReferenceChainAbandoned. No generations=true: + // LivenessTracker::initialize() then clears _gc_generations, so hasLeakSignal() returns true + // without needing any population trend. if ("shouldReportAbandonedSearchOnTinyFrontierCap".equals(testName)) { - return "referencechains=true:hops=32:budget=500:framecap=1:ttl=100:painbudget=100"; + return "memory=64:l:1.0,referencechains=true:hops=32:budget=500:framecap=1:ttl=100:painbudget=100"; } - // Deliberately does not request cpu/wall/memory/nativemem: those categories also - // write an "enabled" ActiveSetting (flightRecorder.cpp:1141-1144) and all default to - // false when not requested, so "enabled"="true" is unambiguous evidence of the - // datadog.ReferenceChain setting specifically without needing to disambiguate by the - // ActiveSetting "id" field (JMC's generic accessor lookup does not resolve that field - // for this custom event type - not worth a bespoke accessor for one assertion). + // shouldExposeReferenceChainsSettingWhenEnabled: a referencechains-only command. Arguments::parse() + // implicitly falls back to cpu profiling, whose ActiveSetting is also "enabled"="true", so the + // test also matches on the ActiveSetting "id" (T_REFERENCE_CHAIN, REFERENCE_CHAIN_EVENT_ID). return "referencechains=true:hops=32:budget=500:ttl=2000:framecap=256"; } @@ -289,8 +298,11 @@ public void shouldExposeReferenceChainsSettingWhenEnabled() { JfrEvents settings = verifyEvents("jdk.ActiveSetting"); boolean sawEnabledSetting = false; for (JfrEvent item : settings) { - if ("enabled".equals(item.getString("name")) && "true".equals(item.getString("value"))) { + if (item.getLong("id", -1) == REFERENCE_CHAIN_EVENT_ID + && "enabled".equals(item.getString("name")) + && "true".equals(item.getString("value"))) { sawEnabledSetting = true; + break; } } assertTrue(sawEnabledSetting, "datadog.ReferenceChain#enabled setting was not found"); @@ -810,15 +822,16 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc * alone was not enough - the search's first pass never ran at all, so it could never reach * {@code SearchState::ABANDONED} for {@code dump()} to observe): {@code LivenessTracker::_gc_generations} * (livenessTracker.cpp's {@code initialize()}) is only ever updated when a {@code start()} command - * actually requests liveness tracking - a bare {@code referencechains=true:...} command like this - * method's own does not call {@code LivenessTracker::start()} at all, so - * {@code gcGenerationsEnabled()} stays stuck at whatever the two {@code generations=true} tests - * above last left it (confirmed directly: this method's own native trace shows - * {@code gc_generations=1}). {@code ReferenceChainTracker::hasLeakSignal()}, which - * {@code shouldRunPass()} consults for even this search's very first pass, therefore falls - * through to {@code selectLeakCandidates()} exactly as the two tests above do - with nothing - * left over from {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}'s own - * {@code resetKlassPopulationForTest0()} call to satisfy it. Seeding + * actually requests liveness tracking. This method's own command now requests + * {@code memory=64:l:1.0} (required for the tracker to start at all, see + * {@code getProfilerCommand()}), so {@code LivenessTracker::initialize()} resets + * {@code gcGenerationsEnabled()} to false and {@code ReferenceChainTracker::hasLeakSignal()} + * returns true unconditionally. The seeding below is kept as defensive insurance in case a + * future command change re-enables generations (then {@code hasLeakSignal()}, which + * {@code shouldRunPass()} consults for even this search's very first pass, falls through to + * {@code selectLeakCandidates()} exactly as the two tests above do - with nothing left over from + * {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}'s own + * {@code resetKlassPopulationForTest0()} call to satisfy it), and is harmless otherwise. Seeding * {@code SEED_EPOCHS_FOR_HYSTERESIS} epochs for a dedicated klass id (see * {@code SEED_EPOCHS_FOR_HYSTERESIS}'s own comment for the full derivation) clears that gate the * same way it does there - this method never sets a representative or polls for it, since it has @@ -853,7 +866,7 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { if ("debug".equals(System.getProperty("ddprof_test.config"))) { // See this method's own javadoc ("Why this method also seeds a klass population trend...") - // for why this is needed even though this method requests no generations=true of its own. + // for why this is kept even though this method requests no generations=true of its own. // // A System.gc() + throwaway dump() first - see this method's own javadoc ("Why the seeding // is preceded by...") for why *both* are needed: a dump() alone triggers cleanup_table() From 2e7ac5d9a30669b7e262deb11795e80bf5d2d541 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 15 Oct 2026 12:43:15 +0200 Subject: [PATCH 10/28] Let the chain-test seam resolve leak-tagged objects via cached chains --- .../src/main/cpp/referenceChainEvents.cpp | 21 ++++++++++++++++++- ddprof-lib/src/main/cpp/referenceChains.h | 18 +++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index 5c2fd1e651..a268dcb0eb 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -500,7 +500,7 @@ void ReferenceChainTracker::invalidateResolvedChain(jlong source_tag) { } jlong ReferenceChainTracker::resolvedChainTargetTagForTest(jlong object_tag) { - if (object_tag <= 0 || isLeakTag(object_tag)) { + if (object_tag <= 0) { return 0; } // _search_start_ns is only rewritten by runPass()/restartSearch() under _engine_lock (BFS thread) @@ -510,6 +510,25 @@ jlong ReferenceChainTracker::resolvedChainTargetTagForTest(jlong object_tag) { // _resolved_chains_lock matches finishLoopIterationSerialized() -> pollWatchedTargets(). MutexLocker engine_guard(_engine_lock); ExclusiveLockGuard guard(&_resolved_chains_lock); + if (isLeakTag(object_tag)) { + // Leak-tag fallback. An object carries its leak tag exactly when no live frontier entry + // exists for it - before the current search's first interception, or after a restart's + // releaseSearchTags() cleared the frontier tags and the next GC re-tagged it - which is + // precisely when the frontier-tag lookup below must miss. A leak tag is minted per tracked + // object from LivenessTracker's pool and released only when that object dies, so unlike a + // frontier tag (which restartSearch() re-mints from 1 for unrelated objects) it identifies + // this exact object across search generations: matching the cached chain event's target + // tag (buildChainEvent() copies the frontier entry's stored leak tag into it) cannot hit a + // different object's chain, so no generation check applies here. restartSearch() keeps + // _resolved_chains, so a chain any earlier search resolved for this object stays reachable + // through this scan even after the frontier tags were wiped. + for (const auto &cached : _resolved_chains) { + if ((jlong)cached.second.event._target_tag == object_tag) { + return object_tag; + } + } + return 0; + } const u64 current_search_ns = load(_search_start_ns); if (current_search_ns == 0) { return 0; diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index 09aaecbbe3..8e872723c9 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -1380,13 +1380,17 @@ class ReferenceChainTracker { int frontierStateForTest(jlong object_tag); // Test seam - not part of the production API. Returns the targetTag a datadog.ReferenceChain - // event for the object currently carrying JVMTI tag `object_tag` would report, but only if the - // CURRENT search has already resolved and cached that object's own chain (_resolved_chains entry - // keyed by its frontier tag with source_search_ns == _search_start_ns). 0 otherwise: untagged, - // leak-tagged but not yet intercepted, no cached chain, or a chain cached by an earlier search - // generation (restartSearch() keeps _resolved_chains and reuses tags from 1, so raw tag equality - // alone could match a stale, unrelated chain). Samples _search_start_ns under _engine_lock and - // _resolved_chains_lock. + // event for the object currently carrying JVMTI tag `object_tag` would report. Two lookups: + // a frontier tag resolves only against the CURRENT search's cached chain (_resolved_chains entry + // keyed by that frontier tag with source_search_ns == _search_start_ns; restartSearch() keeps + // _resolved_chains and re-mints frontier tags from 1, so raw frontier-tag equality across + // generations could match a stale, unrelated chain); a leak tag resolves against any search + // generation's cached chain whose event target tag equals it - a leak tag is minted per + // tracked object and released only when that object dies, so it identifies the same object + // across generations, and the object carries it exactly when no live frontier entry exists + // (never intercepted yet, or a restart cleared the frontier tags and the next GC re-tagged + // it). 0 otherwise: untagged, or no cached chain for that object in any generation. Samples + // _search_start_ns under _engine_lock and _resolved_chains_lock. jlong resolvedChainTargetTagForTest(jlong object_tag); // Test seam - not part of the production API. Evaluates the real private scheduling gate without From c52966f48a37e16de6291176bdcff49b766f219e Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 15 Oct 2026 12:43:17 +0200 Subject: [PATCH 11/28] Deflake ExternalProcessReferenceChainTest static-field scenario Tail now watches a fresh chunk rotated on an 8-attempt maturation window, re-pins the representative each attempt, and raises the safepoint pain budget (painbudget=100) so a loaded box cannot freeze the search engine for the whole tail window. --- .../ExternalProcessReferenceChainTest.java | 11 ++ .../StaticFieldGrowingCollectionScenario.java | 104 ++++++++++++++++-- 2 files changed, 104 insertions(+), 11 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java index a35b61913a..6897334f95 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -162,8 +162,19 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce // (see LeakTagCorrelationReferenceChainTest's own comment for the // full analysis); total volume stays capped by the 256KiB interval // floor, so the tracking table remains tiny. + // painbudget=100 - the safepoint pain budget exists to protect a production + // app from back-to-back expensive passes, but here it starves the engine + // on a loaded box: a cold external JVM's search costs seconds of measured + // safepoint pain, the default cap (5% of wall-clock) clears that debt in + // ~a minute, and the tail then watches a terminal engine with all tags + // released and no new search for its whole window (observed as 1900+ + // consecutive blocked-iteration evaluations and the watched chunk reading + // tag=0 at every poll - the pass/fail line between otherwise identical + // runs). 100% only lets debt clear as fast as it accrues; cadence stays + // bounded by each search's own ~seconds of work, so this cannot spin. String startCommand = "start,memory=64:l:1.0,generations=true," + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:pausetarget=60000" + + ":painbudget=100" + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java index 4b525c4441..c87b5bda2a 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java @@ -69,8 +69,22 @@ private StaticFieldGrowingCollectionScenario() {} private static final int TOTAL_ROUNDS = 25; - // Seeds + lateChunk + one per round: LEAK_BUFFER's elementData never reallocates. - private static final int LEAK_BUFFER_CAPACITY = SEED_CHUNKS + 1 + TOTAL_ROUNDS; + // Tail attempts (below) rotate the watched chunk they append to LEAK_BUFFER, so the backing + // ArrayList's pre-sized capacity must cover them too - see LEAK_BUFFER_CAPACITY's own comment + // for why elementData must never reallocate. + private static final int TAIL_ATTEMPTS = 40; + + // How many consecutive tail attempts the SAME freshly-appended chunk stays the watched target + // before rotating to a new one - see the tail loop's own comment for why a chunk must be given + // several attempts (its discovery matures across walk cycles, slower than one 1.5s attempt) + // and why a chunk that still has not resolved is assumed record-dead and abandoned. + private static final int TAIL_CHUNK_ATTEMPTS = 8; + + // Seeds + lateChunk + one per round + one per tail rotation (sized for the worst case of one + // per attempt so the budgets above can be retuned without touching this): + // LEAK_BUFFER's elementData never reallocates. + private static final int LEAK_BUFFER_CAPACITY = + SEED_CHUNKS + 1 + TOTAL_ROUNDS + TAIL_ATTEMPTS; // FrontierEntryState::EXPANDED (referenceChainFrontier.h), as reported by // JavaProfiler.getReferenceChainFrontierStateForTest0(). @@ -132,9 +146,37 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD throw new IllegalStateException("frontier-state seam returned " + untaggedState + " for an untagged probe object, expected " + FRONTIER_STATE_NONE); } + // Wire a representative for the seeded klass BEFORE the seeds (set-representative-first + // is the load-bearing order: klassPopulationSetRepresentativeForTest() re-keys the + // synthetic entry to the representative's real klass id, and only a representative lets + // the entry mint stable_class_tag). Without a representative topKlassesByGenerationCount() + // skips the entry (no live instance was ever resolved), _watched_leak_klass_count stays + // 0, and runPass() declares the search COMPLETED after every pass - and the terminal + // pass itself then releases every frontier tag (releaseSearchTags()) while + // restartSearch() re-runs the identical walk. The frontier-state seam below blocks on + // _engine_lock mid-pass, so it can only read between passes - where the tags are already + // released - and the EXPANDED wait below can never observe its condition (observed as + // "holders never all reached EXPANDED within 45s" with states [-1, -1, -1]). + // The representative MUST be a byte[] chunk - the leaked klass itself - never the + // list holding it: the representative's real klass becomes the candidate klass, and only + // candidate-klass instances get leak-tagged by tagLeakInstances() and recorded by the + // leak-tag interception path whose chains the seam below reads. A non-byte[] + // representative (e.g. LEAK_BUFFER's own SynchronizedList klass, which has no tracked + // instances in this child JVM) mints no leak tags at all, and lateChunk is never + // discovered (observed as 0/6 runs, the seam returning 0 forever). Watching byte[] does + // flood _leak_parent_fanout with unrelated JVM-wide Object[] parents (~9.5k + // entries/search observed), starving the leak-accumulation rotation prong - but discovery + // does not depend on that prong: the static-anchor descend walk re-walks LEAK_BUFFER's + // whole subtree every search and intercepts every leak-tagged chunk in it (observed + // 10/10 searches), so lateChunk's chain is rebuilt once per search generation regardless. + // The seed chunk is held strongly by the static field, so the tracker's weak + // representative ref stays live. + JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, + LEAK_BUFFER.get(0)); // Seed enough hysteresis-qualifying epochs to unlock ReferenceChainTracker::hasLeakSignal()/ - // shouldRunPass() - WITHOUT wiring in a representative object yet (setKlassPopulationRepresentativeForTest0 - // is not called here). hasLeakSignal() only asks selectLeakCandidates() for a nonzero count - + // shouldRunPass() - AFTER the representative above (set-representative-first is required, + // see that call's own comment). hasLeakSignal() only asks selectLeakCandidates() for a + // nonzero count - // it never reads/dereferences the representative field - so this step's only job is // authorizing the BFS thread's first pass to actually start. Doing this BEFORE waiting on // passesRun() advancing below is load-bearing, found the hard way: a pass only ever runs @@ -219,8 +261,16 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // later change (advance, or reset-to-lower via a restart) proves a walk that started strictly // after lateChunk existed. int passesRunBeforeLateChunk = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; - byte[] lateChunk = new byte[CHUNK_BYTES]; - LEAK_BUFFER.add(lateChunk); + // lateChunk is deliberately NOT held in a local: the main thread's stack is a root-walk root, + // so a live local lets the root enumeration admit (and, once the canary candidate aliases to + // byte[] and a GC runs tagLeakInstances(), leak-tag) lateChunk directly - the resulting + // stack-local-rooted chain never resolves into the static-field chain this scenario watches, + // and the pre-existing tag blocks the elementData re-walk from admitting lateChunk under the + // stale entry at all (observed as buildChainEvent "target_tag not in frontier" for the whole + // run). Re-fetch the chunk from LEAK_BUFFER on every use instead - the only durable path to + // it is the static field, so its discovery proves the stale-entry re-expansion. + final int LATE_CHUNK_INDEX = SEED_CHUNKS; + LEAK_BUFFER.add(new byte[CHUNK_BYTES]); // From here on, seeded incrementally, one epoch per round below, rather than as a single // upfront burst - found the hard way: profiler.execute()'s start() resets @@ -267,7 +317,8 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD LEAK_BUFFER_TEST_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); JavaProfiler.seedTidTrendSample0( LEAK_BUFFER_TEST_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); - JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, lateChunk); + JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, + LEAK_BUFFER.get(LATE_CHUNK_INDEX)); JavaProfiler.pollReferenceChainTargets0(); // Only TRUST a match once passesRun() has genuinely changed since the snapshot taken right @@ -284,7 +335,8 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // tag is read BEFORE dumping - the dump snapshots the same resolved-chain cache - and a // round where the current search has not resolved lateChunk's chain yet is skipped. if (JavaProfiler.referenceChainPassesRunForTest0() != passesRunBeforeLateChunk) { - long lateChunkTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0(lateChunk); + long lateChunkTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0( + LEAK_BUFFER.get(LATE_CHUNK_INDEX)); if (lateChunkTargetTag != 0) { profiler.dump(scratchDumpPath); match = findMatch(scratchDumpPath, lateChunkTargetTag); @@ -311,19 +363,49 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // completed (searchState=COMPLETED) and successfully built the chain. Mirrors // ExternalProcessReferenceChainTest's own comment on why budget/pausetarget are raised this // far for exactly this reason. - for (int attempt = 0; match == null && attempt < 40; attempt++) { + int watchedIndex = -1; + int attemptsOnWatchedChunk = 0; + for (int attempt = 0; match == null && attempt < TAIL_ATTEMPTS; attempt++) { Thread.sleep(1500); // Same passesRun()-advanced check as the round loop above (-DDEBUG builds only: // debug/asan/tsan) - by this point in a real run it has virtually always already flipped, // but there is no reason to drop the guarantee here just because the round loop above // didn't need it. - // Same lateChunk-identity pinning as the round loop above. long requiredTargetTag = 0; if (debugBuild) { if (JavaProfiler.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { continue; } - requiredTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0(lateChunk); + // The tail watches, one at a time, chunks appended to LEAK_BUFFER well after that + // collection was first EXPANDED - exactly the discovery this scenario exists to prove. + // Watching a chunk allocated once at scenario start (lateChunk above) is not durable: + // the whole leak-tag pipeline keys on the chunk's allocation-tracking record, records + // are only minted at allocation, and the tracking table's churn can evict an old + // chunk's record for the rest of the run - observed as the representative resolving + // that chunk correctly but reading tag=0 for 2000+ consecutive polls (no leak tag ever + // minted, so no interception and no chain ever built for it). A FRESHLY appended chunk + // has a guaranteed-live record: the explicit poll below tags it immediately, the next + // static-anchor descend intercepts it, and a later attempt's poll resolves its chain. + // But discovery takes several walk cycles (interception, then a later poll's chain + // build) - measured at more than one 1.5s attempt - so the SAME chunk must stay the + // watched target for several attempts before rotating to a new one; rotating every + // attempt reads a just-allocated chunk that has not matured yet and abandons each + // chunk right before it would resolve (observed as 0/8 runs). If a chunk still has + // not resolved after TAIL_CHUNK_ATTEMPTS attempts, its record most likely died the + // same death lateChunk's can - rotate. Same no-local discipline as lateChunk above: + // each watched chunk is reachable only through LEAK_BUFFER, so any walk that reaches + // it comes through elementData, never through a stack-local root. + if (watchedIndex < 0 || attemptsOnWatchedChunk >= TAIL_CHUNK_ATTEMPTS) { + LEAK_BUFFER.add(new byte[CHUNK_BYTES]); + watchedIndex = LEAK_BUFFER.size() - 1; + attemptsOnWatchedChunk = 0; + } + attemptsOnWatchedChunk++; + JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, + LEAK_BUFFER.get(watchedIndex)); + JavaProfiler.pollReferenceChainTargets0(); + requiredTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0( + LEAK_BUFFER.get(watchedIndex)); if (requiredTargetTag == 0) { continue; } From b4dd630dc2ae25ec2313b86fcfd52edbe75fff05 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 15 Oct 2026 17:17:29 +0200 Subject: [PATCH 12/28] sphinx: address review feedback on PR #800 --- ddprof-lib/src/main/cpp/event.h | 7 +- ddprof-lib/src/main/cpp/livenessTracker.cpp | 14 +- ddprof-lib/src/main/cpp/livenessTracker.h | 44 ++++- .../src/main/cpp/referenceChainEvents.cpp | 46 +++-- .../src/main/cpp/referenceChainFrontier.cpp | 1 + .../src/main/cpp/referenceChainFrontier.h | 21 +- .../src/main/cpp/referenceChainTraversal.cpp | 4 +- .../src/main/cpp/referenceChainWalk.cpp | 5 +- ddprof-lib/src/main/cpp/referenceChains.h | 29 ++- .../com/datadoghq/profiler/JavaProfiler.java | 7 +- .../src/test/cpp/livenessTracker_ut.cpp | 81 ++++++++ .../test/cpp/referenceChainsAnchorTests.inc | 5 +- .../test/cpp/referenceChainsEventTests.inc | 45 ++++- .../test/cpp/referenceChainsTestAccessors.h | 11 ++ .../test/cpp/referenceChainsTrackerTests.inc | 99 ++++++++++ .../test/fuzz/fuzz_referenceChainFrontier.cpp | 2 +- .../profiler/AbstractProcessProfilerTest.java | 13 ++ ...15-leak-tag-reuse-cached-chain-identity.md | 182 ++++++++++++++++++ 18 files changed, 583 insertions(+), 33 deletions(-) create mode 100644 docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md diff --git a/ddprof-lib/src/main/cpp/event.h b/ddprof-lib/src/main/cpp/event.h index cf6c90d342..d87f40d672 100644 --- a/ddprof-lib/src/main/cpp/event.h +++ b/ddprof-lib/src/main/cpp/event.h @@ -139,10 +139,15 @@ class ReferenceChainEvent : public Event { u64 _target_tag; u32 _depth; u8 _root_kind; + // In-memory only, never serialized: LivenessTracker's ownership-period stamp of _target_tag when + // it is a leak tag, captured when the tag was bound to the target's frontier entry. Leak tags are + // recycled, so (_target_tag, _leak_tag_incarnation) identifies the target; 0 otherwise. + u32 _leak_tag_incarnation; std::vector _hops; ReferenceChainEvent() - : Event(), _start_time(0), _target_tag(0), _depth(0), _root_kind(0) {} + : Event(), _start_time(0), _target_tag(0), _depth(0), _root_kind(0), + _leak_tag_incarnation(0) {} }; // Search-level abandonment signal: reports why the reference-chain search diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp index a3a1a6afe1..484650fec4 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ b/ddprof-lib/src/main/cpp/livenessTracker.cpp @@ -451,13 +451,17 @@ jlong LivenessTracker::acquireLeakTag() { // other, but any future second shared-lock mutator would corrupt the LIFO // free list. The dedicated lock keeps the pool correct independent of which // table lock mode the caller holds. Lock order: _table_lock (any mode) is - // always acquired BEFORE _leak_tag_pool_lock, never the reverse. + // always acquired BEFORE _leak_tag_pool_lock, never the reverse. The same + // lock serializes the ownership-period stamp writes (_leak_tag_incarnation), + // which are bumped on every ownership transition so a reader can tell this + // owner of the tag from an earlier, dead one; stamp reads are lock-free. ExclusiveLockGuard guard(&_leak_tag_pool_lock); if (_leak_tag_free_count <= 0) { return 0; // pool exhausted } int idx = _leak_tag_free_list[--_leak_tag_free_count]; _leak_tag_in_use[idx] = true; + bumpLeakTagIncarnationLocked(idx); // free -> in use: a new ownership period return LEAK_TAG_BASE + idx; } @@ -476,6 +480,9 @@ void LivenessTracker::releaseLeakTag(jlong tag) { return; } _leak_tag_in_use[idx] = false; + // in use -> free: chains captured during the ending ownership period no longer + // describe whoever owns the tag next. + bumpLeakTagIncarnationLocked(idx); _leak_tag_free_list[_leak_tag_free_count++] = idx; } @@ -2039,6 +2046,11 @@ Error LivenessTracker::start(Arguments &args) { _leak_tag_pool_lock.lock(); int free_w = 0; for (int i = 0; i < LEAK_TAG_POOL_SIZE; i++) { + if (_leak_tag_in_use[i] != tag_owned[i]) { + // Ownership of this slot changes in the rebuild; a slot whose owner is + // unchanged keeps its stamp, so its cached chains stay attributable. + bumpLeakTagIncarnationLocked(i); + } _leak_tag_in_use[i] = tag_owned[i]; if (!tag_owned[i]) { _leak_tag_free_list[free_w++] = i; diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index 84fce37cff..d49972e778 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -571,13 +571,34 @@ class alignas(alignof(SpinLock)) LivenessTracker { // holder excludes the exclusive one, but only this dedicated lock makes the // pool safe for any future second shared-lock mutator. Lock order: _table_lock // (any mode) is always acquired BEFORE _leak_tag_pool_lock, never the - // reverse; no path acquires _table_lock while holding the pool lock. + // reverse; no path acquires _table_lock while holding the pool lock. It also + // serializes every write of _leak_tag_incarnation (reads of the stamp take no + // lock - see leakTagIncarnation()). mutable SpinLock _leak_tag_pool_lock; int _leak_tag_free_list[LEAK_TAG_POOL_SIZE]; int _leak_tag_free_count; // Per-tag ownership: set by acquireLeakTag(), cleared by releaseLeakTag(), // so a repeated release of a tag that was not re-acquired is ignored. bool _leak_tag_in_use[LEAK_TAG_POOL_SIZE]; + // Per-slot ownership-period stamp. A leak tag is recycled once its object + // dies, so the tag alone does not identify one object: (tag, stamp) does. + // The stamp changes on every ownership transition of the slot - acquire + // (free -> in use), a real release (in use -> free), and every slot whose + // in-use bit flips in start()'s rebuild or in leakTagPoolResetForTest() - + // and never on a no-op (rejected or out-of-range release, unchanged slot). + // Written only under _leak_tag_pool_lock (atomic release stores), read + // lock-free with acquire semantics so heap callbacks and reference-chain + // code never take the pool lock. The u32 wraps after 2^32 transitions of one + // slot; a false "same owner" needs exactly that many transitions between a + // stamp's capture and its check, which is accepted as negligible. + u32 _leak_tag_incarnation[LEAK_TAG_POOL_SIZE]; + + // Advances slot `idx`'s ownership-period stamp. Caller holds + // _leak_tag_pool_lock. + void bumpLeakTagIncarnationLocked(int idx) { + __atomic_store_n(&_leak_tag_incarnation[idx], + _leak_tag_incarnation[idx] + 1, __ATOMIC_RELEASE); + } jlong acquireLeakTag(); void releaseLeakTag(jlong tag); @@ -834,7 +855,21 @@ class alignas(alignof(SpinLock)) LivenessTracker { _gc_generations(false), _klass_population_size(0), _klass_count_scratch_size(0), _last_class_map_generation(0), - _leak_tag_free_count(LEAK_TAG_POOL_SIZE) {} + _leak_tag_free_count(LEAK_TAG_POOL_SIZE), + _leak_tag_incarnation() {} // value-initialized: every stamp starts at 0 + + // Ownership-period stamp of leak tag `tag` (see _leak_tag_incarnation), or 0 + // when `tag` is outside the pool range. Lock-free (one acquire load): safe + // from heap callbacks and while holding reference-chain locks. A cached + // reference chain whose captured stamp equals this value was built for the + // tag's current owner. + u32 leakTagIncarnation(jlong tag) const { + if (tag < LEAK_TAG_BASE || tag >= LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE) { + return 0; + } + return __atomic_load_n(&_leak_tag_incarnation[tag - LEAK_TAG_BASE], + __ATOMIC_ACQUIRE); + } // Reset recording-scoped gates at every accepted profiler start. Persistent tracking and // population state intentionally survives recording boundaries. @@ -1075,8 +1110,13 @@ class alignas(alignof(SpinLock)) LivenessTracker { // tagLeakInstances() itself needs a live JVM (SetTag/NewLocalRef) and stays // out of gtest's reach. void leakTagPoolResetForTest() { + ExclusiveLockGuard guard(&_leak_tag_pool_lock); for (int i = 0; i < LEAK_TAG_POOL_SIZE; i++) { _leak_tag_free_list[i] = i; + if (_leak_tag_in_use[i]) { + // Freeing an owned slot is an ownership transition. + bumpLeakTagIncarnationLocked(i); + } _leak_tag_in_use[i] = false; } _leak_tag_free_count = LEAK_TAG_POOL_SIZE; diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index a268dcb0eb..e55ad2565c 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -105,6 +105,9 @@ bool ReferenceChainTracker::buildChainEvent(jvmtiEnv *jvmti, JNIEnv *jni, (long long)target_tag, chain.size(), chain.empty() ? 0u : chain[0], entry.depth, (unsigned)root_kind, (long long)entry.leak_tag); out->_target_tag = entry.leak_tag != 0 ? (u64)entry.leak_tag : (u64)target_tag; + // Ownership period of the leak tag at binding time (0 without a leak tag); see + // chainTargetsCurrentLeakOwner(). + out->_leak_tag_incarnation = entry.leak_tag_incarnation; out->_depth = entry.depth; out->_root_kind = root_kind; out->_hops.resize(chain.size()); @@ -514,16 +517,20 @@ jlong ReferenceChainTracker::resolvedChainTargetTagForTest(jlong object_tag) { // Leak-tag fallback. An object carries its leak tag exactly when no live frontier entry // exists for it - before the current search's first interception, or after a restart's // releaseSearchTags() cleared the frontier tags and the next GC re-tagged it - which is - // precisely when the frontier-tag lookup below must miss. A leak tag is minted per tracked - // object from LivenessTracker's pool and released only when that object dies, so unlike a - // frontier tag (which restartSearch() re-mints from 1 for unrelated objects) it identifies - // this exact object across search generations: matching the cached chain event's target - // tag (buildChainEvent() copies the frontier entry's stored leak tag into it) cannot hit a - // different object's chain, so no generation check applies here. restartSearch() keeps - // _resolved_chains, so a chain any earlier search resolved for this object stays reachable - // through this scan even after the frontier tags were wiped. + // precisely when the frontier-tag lookup below must miss. A leak tag is NOT an object + // identity on its own: LivenessTracker releases it when its object dies and recycles it for + // another object, while _resolved_chains keeps the dead object's chain (restartSearch() keeps + // the cache too). Identity is the pair (leak tag, ownership-period stamp): buildChainEvent() + // copies the frontier entry's leak tag and the stamp captured when that tag was bound, and + // chainTargetsCurrentLeakOwner() accepts the chain only if the stamp is still the tag's + // current one. Within one ownership period no generation check applies, so a chain any + // earlier search resolved for this object stays reachable through this scan even after the + // frontier tags were wiped; a chain cached for an earlier owner of the tag never matches. The + // caller holds a strong reference to the target, so its own tag cannot be released (and its + // stamp cannot change) during this call. for (const auto &cached : _resolved_chains) { - if ((jlong)cached.second.event._target_tag == object_tag) { + if ((jlong)cached.second.event._target_tag == object_tag && + chainTargetsCurrentLeakOwner(cached.second.event)) { return object_tag; } } @@ -779,7 +786,10 @@ bool ReferenceChainTracker::correlateAdmittedLeakTag(jlong frontier_tag, // re-admission. return true; } - _frontier->setLeakTag(frontier_tag, leak_tag); + // The caller's row owns leak_tag now (tagLeakInstances() holds the shared _table_lock, which + // excludes cleanup_table()'s release), so this is the owner's ownership-period stamp. + _frontier->setLeakTag(frontier_tag, leak_tag, + LivenessTracker::instance()->leakTagIncarnation(leak_tag)); TEST_LOG_SUMMARY("ReferenceChainTracker::correlateAdmittedLeakTag " "frontier_tag=%lld leak_tag=%lld depth=%u parent_tag=%lld", (long long)frontier_tag, (long long)leak_tag, entry.depth, @@ -789,6 +799,12 @@ bool ReferenceChainTracker::correlateAdmittedLeakTag(jlong frontier_tag, return true; } +bool ReferenceChainTracker::chainTargetsCurrentLeakOwner(const ReferenceChainEvent &e) { + jlong target = (jlong)e._target_tag; + return isLeakTag(target) && + e._leak_tag_incarnation == LivenessTracker::instance()->leakTagIncarnation(target); +} + void ReferenceChainTracker::drainPendingChainEvents( std::vector *out) { if (out == nullptr) { @@ -803,7 +819,15 @@ void ReferenceChainTracker::drainPendingChainEvents( { ExclusiveLockGuard guard(&_resolved_chains_lock); for (const auto &kv : _resolved_chains) { - out->push_back(kv.second.event); + const ReferenceChainEvent &event = kv.second.event; + if (isLeakTag((jlong)event._target_tag) && !chainTargetsCurrentLeakOwner(event)) { + // The leak-tagged target died (its tag was released, and possibly re-acquired by another + // object): emitting targetTag = L would let the consumer's ReferenceChain.targetTag -> + // HeapLiveObject.leakTag join attribute this chain to L's current owner. Skip it, but do + // not erase it - the existing eviction policy decides when it leaves the cache. + continue; + } + out->push_back(event); out->back()._start_time = now; } } diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp b/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp index d56044658c..0cef418bd3 100644 --- a/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp @@ -115,6 +115,7 @@ bool FrontierTable::insert(jlong tag, jlong parent_tag, u32 referrer_klass, _table[idx].root_kind = root_kind; _table[idx].class_tag = class_tag; _table[idx].leak_tag = 0; + _table[idx].leak_tag_incarnation = 0; _table[idx].referrer_field_index = referrer_field_index; _table[idx].edge_kind = edge_kind; _table[idx].referrer_class_tag = referrer_class_tag; diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.h b/ddprof-lib/src/main/cpp/referenceChainFrontier.h index 9228b516e6..4c2ca19964 100644 --- a/ddprof-lib/src/main/cpp/referenceChainFrontier.h +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.h @@ -55,6 +55,12 @@ typedef struct FrontierEntry { u32 referrer_klass; // StringDictionary id, 0 = unresolved/none u32 depth; // hop count from the frontier's seed, for the hop cap u8 state; // one of FrontierEntryState's constants + // LivenessTracker::leakTagIncarnation(leak_tag) captured when leak_tag was bound to this entry + // (setLeakTag()): the ownership period of the tag this entry's object held. A leak tag is + // recycled after its object dies, so (leak_tag, leak_tag_incarnation) - not leak_tag alone - + // identifies the object. 0 while leak_tag == 0. Sits in the padding after `state`, so + // sizeof(FrontierEntry) is unchanged (static_assert below). + u32 leak_tag_incarnation; // The leak tag assigned by LivenessTracker to this specific tracked object, copied from the JVMTI // tag at admission time. jlong leak_tag; @@ -88,6 +94,9 @@ typedef struct FrontierEntry { jlong referrer_class_tag; } FrontierEntry; +static_assert(sizeof(FrontierEntry) == 64, + "FrontierEntry::leak_tag_incarnation must stay in the padding after `state`"); + // Per-hop retention-edge identity collected by reconstructChain() alongside the class chain: // everything needed at emission to label HOW chain[i] is retained (the field of chain[i+1] pointing // at it, or the root edge for the last hop). @@ -196,8 +205,15 @@ class alignas(alignof(SpinLock)) FrontierTable { void updateRootKind(jlong tag, u8 root_kind); // Set the leak tag on a frontier entry (the JVMTI tag assigned by LivenessTracker to this - // specific tracked leaking object). - void setLeakTag(jlong tag, jlong leak_tag) { + // specific tracked leaking object) together with that tag's ownership-period stamp + // (LivenessTracker::leakTagIncarnation(leak_tag)) read at the binding site. The stamp is required + // (no default): a forgotten 0 would make drainPendingChainEvents() drop the live object's chain. + // The stamp read at each binding site is the current owner's: heapReferenceCallback() and + // heapRootCallback() intercept the object live inside FollowReferences, so cleanup_table() + // cannot release its tag concurrently; correlateAdmittedLeakTag() runs under + // LivenessTracker::tagLeakInstances()' shared _table_lock, which excludes cleanup_table()'s + // release. + void setLeakTag(jlong tag, jlong leak_tag, u32 leak_tag_incarnation) { if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { return; } @@ -205,6 +221,7 @@ class alignas(alignof(SpinLock)) FrontierTable { ExclusiveLockGuard guard(&_table_lock); if (idx < _table_size) { _table[idx].leak_tag = leak_tag; + _table[idx].leak_tag_incarnation = leak_tag_incarnation; } } diff --git a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp index 8d6e87d9d9..f9c1207890 100644 --- a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp @@ -91,7 +91,9 @@ jvmtiIterationControl JNICALL ReferenceChainTracker::heapRootCallback( if (ctx->frontier->insert(frontier_tag, /*parent_tag=*/0, referrer_klass, /*depth=*/0, FrontierEntryState::FRONTIER, translated_root_kind, class_tag)) { - ctx->frontier->setLeakTag(frontier_tag, leak_tag); + // Capture the current ownership period, as in heapReferenceCallback()'s interception. + ctx->frontier->setLeakTag(frontier_tag, leak_tag, + LivenessTracker::instance()->leakTagIncarnation(leak_tag)); *tag_ptr = frontier_tag; ctx->edges_admitted++; ctx->tracker->trackLeakAccumulation(ctx->frontier, class_tag, 0, diff --git a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp index 320e9048b9..63f99f4c0e 100644 --- a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp @@ -296,7 +296,10 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( (u8)reference_kind, parent_tag == 0 ? edge_referrer_class_tag : 0)) { // Store the leak tag in the frontier entry - ctx->frontier->setLeakTag(frontier_tag, leak_tag); + // Capture the tag's current ownership period with it (one lock-free load): this live object + // owns leak_tag now, so a chain built from this entry describes it and not an earlier owner. + ctx->frontier->setLeakTag(frontier_tag, leak_tag, + LivenessTracker::instance()->leakTagIncarnation(leak_tag)); *tag_ptr = frontier_tag; ctx->edges_admitted++; // Queue for expandFrontier() like admitObject() does: FollowReferences usually descends into diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index 8e872723c9..a24e6f1213 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -1044,6 +1044,14 @@ class ReferenceChainTracker { // comment for why this cannot be deferred to dump()-time). void enqueuePendingAbandonedEvent(); + // True iff `e` targets a leak tag AND was built during that tag's current ownership period: + // e._leak_tag_incarnation == LivenessTracker::leakTagIncarnation(e._target_tag). Leak tags are + // recycled once their object dies, while _resolved_chains keeps the dead object's chain, so tag + // equality alone would attribute it to the tag's next owner. False for a non-leak (frontier-tag) + // target - each caller decides what that means for it. Lock-free (one acquire load); never takes + // LivenessTracker's pool lock, so it is safe under _resolved_chains_lock. + static bool chainTargetsCurrentLeakOwner(const ReferenceChainEvent &e); + public: static ReferenceChainTracker *instance() { static ReferenceChainTracker instance; @@ -1331,7 +1339,11 @@ class ReferenceChainTracker { // Appends a copy of every currently-cached resolved chain to *out, re-stamped with a fresh // _start_time so it lands in the dumping chunk's time window, WITHOUT clearing the cache - a // repeatable snapshot, not a drain, so the same live sample's chain is re-emitted into every JFR - // chunk it survives into (see _resolved_chains' own comment). + // chunk it survives into (see _resolved_chains' own comment). Exception: a chain whose target is + // a leak tag but which fails chainTargetsCurrentLeakOwner() (its object died and the tag may have + // a new owner) is not emitted - a consumer joins ReferenceChain.targetTag to + // HeapLiveObject.leakTag, which would attribute it to the new owner. Such a chain is not erased + // either; the existing cache eviction policy still decides when it leaves. void drainPendingChainEvents(std::vector *out); // Appends every abandoned-search event queued since the last call and clears the queue - a true @@ -1385,12 +1397,15 @@ class ReferenceChainTracker { // keyed by that frontier tag with source_search_ns == _search_start_ns; restartSearch() keeps // _resolved_chains and re-mints frontier tags from 1, so raw frontier-tag equality across // generations could match a stale, unrelated chain); a leak tag resolves against any search - // generation's cached chain whose event target tag equals it - a leak tag is minted per - // tracked object and released only when that object dies, so it identifies the same object - // across generations, and the object carries it exactly when no live frontier entry exists - // (never intercepted yet, or a restart cleared the frontier tags and the next GC re-tagged - // it). 0 otherwise: untagged, or no cached chain for that object in any generation. Samples - // _search_start_ns under _engine_lock and _resolved_chains_lock. + // generation's cached chain whose event target tag equals it AND whose captured ownership-period + // stamp equals the tag's current one (chainTargetsCurrentLeakOwner()). A leak tag is released + // when its object dies and then recycled for another object, so leak-tag identity is the pair + // (leak tag, ownership-period stamp), not the tag alone; within one ownership period it does + // identify the same object across search generations, and the object carries it exactly when + // no live frontier entry exists (never intercepted yet, or a restart cleared the frontier tags + // and the next GC re-tagged it). 0 otherwise: untagged, or no cached chain for that object's + // current ownership period in any generation. Samples _search_start_ns under _engine_lock and + // _resolved_chains_lock. jlong resolvedChainTargetTagForTest(jlong object_tag); // Test seam - not part of the production API. Evaluates the real private scheduling gate without diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 5c341214a6..d8c84df2fb 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -685,8 +685,11 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l /** * Test seam (-DDEBUG native builds only - debug/asan/tsan; returns 0 in release): does not * tag {@code target}. The targetTag the {@code datadog.ReferenceChain} event for target - * carries, if the current search has already resolved and cached target's own chain; 0 - * otherwise. + * carries, if a cached chain exists for target; 0 otherwise. When target carries a frontier + * tag, only the current search's cached chain counts. When target carries a leak tag, a chain + * cached by any search generation counts, but only if it was built during target's current + * ownership of that leak tag: leak tags are recycled after their object dies, so identity is + * the (leak tag, ownership-period stamp) pair, not the tag alone. */ public static native long getReferenceChainResolvedTargetTagForTest0(Object target); diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp index cb59c87ca5..cb79e49098 100644 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp @@ -1560,6 +1560,87 @@ TEST_F(LeakTagPoolTest, DoubleReleaseWithFreeRoomListsTagOnce) { EXPECT_NE(first, second) << "the same tag was handed out twice"; } +// Ownership-period stamp (leakTagIncarnation): (tag, stamp) is the identity of +// a leak-tagged object, because the tag itself is recycled after its object +// dies. The stamp must change on every ownership transition and on nothing +// else. +TEST_F(LeakTagPoolTest, IncarnationChangesOnAcquireAndRelease) { + LivenessTracker *tracker = LivenessTracker::instance(); + jlong base = tracker->leakTagBaseForTest(); + // LIFO: after the reset the next acquire pops this slot. + jlong tag = tracker->acquireLeakTagForTest(); + ASSERT_GE(tag, base); + u32 owned = tracker->leakTagIncarnation(tag); + tracker->releaseLeakTagForTest(tag); + u32 released = tracker->leakTagIncarnation(tag); + EXPECT_NE(owned, released) << "release must start a new ownership period"; + jlong re_tag = tracker->acquireLeakTagForTest(); + ASSERT_EQ(tag, re_tag) << "released tag should be recycled first (LIFO)"; + u32 reowned = tracker->leakTagIncarnation(re_tag); + EXPECT_NE(released, reowned) << "acquire must start a new ownership period"; + EXPECT_NE(owned, reowned) + << "the tag's new owner must not share the dead owner's stamp"; +} + +TEST_F(LeakTagPoolTest, IncarnationUnchangedByNoOpReleases) { + LivenessTracker *tracker = LivenessTracker::instance(); + jlong base = tracker->leakTagBaseForTest(); + int pool_size = tracker->leakTagPoolSizeForTest(); + jlong tag = tracker->acquireLeakTagForTest(); + ASSERT_GE(tag, base); + tracker->releaseLeakTagForTest(tag); + u32 after_release = tracker->leakTagIncarnation(tag); + // Rejected repeated release: no ownership transition. + tracker->releaseLeakTagForTest(tag); + EXPECT_EQ(after_release, tracker->leakTagIncarnation(tag)); + + // Out-of-range releases touch no slot. + jlong held = tracker->acquireLeakTagForTest(); + ASSERT_GE(held, base); + u32 held_stamp = tracker->leakTagIncarnation(held); + u32 first_stamp = tracker->leakTagIncarnation(base); + u32 last_stamp = tracker->leakTagIncarnation(base + pool_size - 1); + tracker->releaseLeakTagForTest(base - 1); + tracker->releaseLeakTagForTest(base + pool_size); + tracker->releaseLeakTagForTest(0); + tracker->releaseLeakTagForTest(-1); + EXPECT_EQ(held_stamp, tracker->leakTagIncarnation(held)); + EXPECT_EQ(first_stamp, tracker->leakTagIncarnation(base)); + EXPECT_EQ(last_stamp, tracker->leakTagIncarnation(base + pool_size - 1)); +} + +TEST_F(LeakTagPoolTest, IncarnationIsZeroOutOfRange) { + LivenessTracker *tracker = LivenessTracker::instance(); + jlong base = tracker->leakTagBaseForTest(); + int pool_size = tracker->leakTagPoolSizeForTest(); + // Bump an in-range stamp first, so out-of-range reads cannot pass by + // accidentally aliasing a still-zero slot. + jlong tag = tracker->acquireLeakTagForTest(); + ASSERT_GE(tag, base); + EXPECT_EQ(0u, tracker->leakTagIncarnation(0)); + EXPECT_EQ(0u, tracker->leakTagIncarnation(1)); + EXPECT_EQ(0u, tracker->leakTagIncarnation(-1)); + EXPECT_EQ(0u, tracker->leakTagIncarnation(base - 1)); + EXPECT_EQ(0u, tracker->leakTagIncarnation(base + pool_size)); +} + +TEST_F(LeakTagPoolTest, ResetForTestBumpsOnlyOwnedSlots) { + LivenessTracker *tracker = LivenessTracker::instance(); + jlong base = tracker->leakTagBaseForTest(); + jlong owned = tracker->acquireLeakTagForTest(); + jlong freed = tracker->acquireLeakTagForTest(); + ASSERT_GE(owned, base); + ASSERT_GE(freed, base); + tracker->releaseLeakTagForTest(freed); + u32 owned_before = tracker->leakTagIncarnation(owned); + u32 freed_before = tracker->leakTagIncarnation(freed); + tracker->leakTagPoolResetForTest(); + EXPECT_NE(owned_before, tracker->leakTagIncarnation(owned)) + << "freeing an owned slot is an ownership transition"; + EXPECT_EQ(freed_before, tracker->leakTagIncarnation(freed)) + << "an already-free slot keeps its stamp"; +} + // --------------------------------------------------------------------------- // C1 chaos: concurrent acquire/release across forced epoch rotations. The // single-threaded tests above pin the sequential mechanics; this one attacks diff --git a/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc b/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc index af0dbdf434..691b0f41f4 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc @@ -110,7 +110,7 @@ TEST_F(ReferenceChainsBfsTest, ContainerAnchorLeapsQueueAcrossLargeIndex) { ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( frontier, leak_anchor, 0, 0, FrontierEntryState::FRONTIER, JVMTI_HEAP_REFERENCE_STATIC_FIELD)); - frontier->setLeakTag(leak_anchor, 777); + frontier->setLeakTag(leak_anchor, 777, /*leak_tag_incarnation=*/0); ReferenceChainsTestAccessor::addToStaticAnchorIndexForTest( leak_anchor, 599999, JVMTI_HEAP_REFERENCE_STATIC_FIELD); ReferenceChainsTestAccessor::primeClassShapeForTest( @@ -332,7 +332,8 @@ TEST_F(ReferenceChainsBfsTest, LeakTagChainMarksCanaryFound) { // (suppressChainEvent: depth==0, or depth==1 at a transient root) keeps both chains. ASSERT_TRUE(frontier->insert(4242, 0, 7, 1, FrontierEntryState::FRONTIER, 8, /*class_tag=*/0, -1, /*kind=*/0)); - frontier->setLeakTag(4242, 1073742079); + frontier->setLeakTag(4242, 1073742079, + LivenessTracker::instance()->leakTagIncarnation(1073742079)); ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest(7, 4242, true); ASSERT_EQ(4242, ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); ASSERT_TRUE(frontier->insert(5353, 0, 9, 1, FrontierEntryState::FRONTIER, diff --git a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc index 2559036d1c..42af0c17a2 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc @@ -358,13 +358,17 @@ TEST_F(ReferenceChainsBfsTest, RecordDiscoveredInstanceEvictsNoiseSlots) { // signal). ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( frontier, 200, 0, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); - frontier->setLeakTag(200, ReferenceChainsTestAccessor::leakTagBase() + 1); + frontier->setLeakTag(200, ReferenceChainsTestAccessor::leakTagBase() + 1, + LivenessTracker::instance()->leakTagIncarnation( + ReferenceChainsTestAccessor::leakTagBase() + 1)); // Entries for the remaining noise slots so the eviction scan finds all slots leak-tagged. for (int d = 1; d < cap; d++) { jlong tag = 101 + (d - 1); ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( frontier, tag, 0, 1, FrontierEntryState::EXPANDED, /*root_kind=*/0)); - frontier->setLeakTag(tag, ReferenceChainsTestAccessor::leakTagBase() + 2); + frontier->setLeakTag(tag, ReferenceChainsTestAccessor::leakTagBase() + 2, + LivenessTracker::instance()->leakTagIncarnation( + ReferenceChainsTestAccessor::leakTagBase() + 2)); } ReferenceChainsTestAccessor::recordDiscoveredInstanceForTest( kKlass, 201, true); @@ -398,6 +402,9 @@ TEST_F(ReferenceChainsBfsTest, CorrelateAdmittedLeakTagSetsEntryAndDiscovers) { EXPECT_TRUE(tracker->correlateAdmittedLeakTag(300, kLeakTag, kKlass)); EXPECT_EQ(kLeakTag, (jlong)ReferenceChainsTestAccessor::frontierLeakTag(300)); + // The binding captures the tag's current ownership-period stamp with it. + EXPECT_EQ(LivenessTracker::instance()->leakTagIncarnation(kLeakTag), + ReferenceChainsTestAccessor::frontierLeakTagIncarnation(300)); EXPECT_EQ(1, ReferenceChainsTestAccessor::candidateDiscoveredCountForTest(0)); EXPECT_EQ((jlong)300, ReferenceChainsTestAccessor::candidateDiscoveredTagForTest(0, 0)); @@ -413,6 +420,40 @@ TEST_F(ReferenceChainsBfsTest, CorrelateAdmittedLeakTagSetsEntryAndDiscovers) { tracker->stop(); } +// The ownership-period stamp bound with a leak tag travels with the entry into the chain event +// (buildChainEvent()), and a fresh insert() into the slot clears it together with the leak tag. +TEST_F(ReferenceChainsBfsTest, BuildChainEventPropagatesLeakTagIncarnation) { + Arguments args; + ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); + ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); + ASSERT_FALSE(tracker->start(args)); + FrontierTable *frontier = tracker->frontierTable(); + + constexpr jlong kLeakTag = 0x40000000LL + 7; + constexpr u32 kStamp = 12345u; + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 400, 0, 0, FrontierEntryState::EXPANDED, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::frontierLeakTagIncarnation(400)); + frontier->setLeakTag(400, kLeakTag, kStamp); + EXPECT_EQ(kStamp, ReferenceChainsTestAccessor::frontierLeakTagIncarnation(400)); + + ReferenceChainEvent event; + ASSERT_TRUE(tracker->buildChainEvent(&mock_jvmti, &mock_jni, 400, &event)); + EXPECT_EQ((u64)kLeakTag, event._target_tag); + EXPECT_EQ(kStamp, event._leak_tag_incarnation); + + // Re-inserting the slot (a new object under that frontier tag) drops both the leak tag and its + // stamp. + ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( + frontier, 400, 0, 0, FrontierEntryState::FRONTIER, + JVMTI_HEAP_REFERENCE_STATIC_FIELD)); + EXPECT_EQ(0, ReferenceChainsTestAccessor::frontierLeakTag(400)); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::frontierLeakTagIncarnation(400)); + + tracker->stop(); +} + // Retention-explanation gate on the discovered-instance chains (depth==0 always suppressed; // depth==1 suppressed only for TRANSIENT roots - a depth-1 chain from a durable root is the real // direct-retention shape): transient depth-1 must NOT be cached, durable depth-1 and depth-2 must. diff --git a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h index 22d345521e..5445b18eec 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h +++ b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h @@ -313,6 +313,17 @@ class ReferenceChainsTestAccessor { return entry.leak_tag; } + // Ownership-period stamp captured with the entry's leak tag (FrontierEntry:: + // leak_tag_incarnation), or 0 when the entry does not exist. + static u32 frontierLeakTagIncarnation(jlong tag) { + ReferenceChainTracker *t = ReferenceChainTracker::instance(); + FrontierEntry entry{}; + if (t->_frontier == nullptr || !t->_frontier->lookup(tag, &entry)) { + return 0; + } + return entry.leak_tag_incarnation; + } + static void setCandidateKlassIdForTest(int idx, u32 klass_id) { ReferenceChainTracker::instance()->setCandidateKlassIdForTest(idx, klass_id); } diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc index 3f9124d7d6..6528994dc1 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -473,10 +473,32 @@ class ResolvedChainCacheTest : public ::testing::Test { protected: void SetUp() override { ReferenceChainsTestAccessor::reset(); + // Leak-tag ownership-period tests below drive the shared pool; keep its state isolated. + LivenessTracker::instance()->leakTagPoolResetForTest(); } void TearDown() override { ReferenceChainsTestAccessor::reset(); + LivenessTracker::instance()->leakTagPoolResetForTest(); + } + + // A cached chain targeting leak tag `leak_tag`, stamped with the ownership period `stamp` + // (what buildChainEvent() copies from the frontier entry). + static ReferenceChainEvent makeLeakEvent(jlong leak_tag, u32 stamp) { + ReferenceChainEvent event = makeEvent((u64)leak_tag); + event._leak_tag_incarnation = stamp; + return event; + } + + static bool drainContainsTarget(u64 target_tag) { + std::vector out; + ReferenceChainsTestAccessor::drain(&out); + for (const auto &e : out) { + if (e._target_tag == target_tag) { + return true; + } + } + return false; } static ReferenceChainEvent makeEvent(u64 target_tag) { @@ -843,6 +865,83 @@ TEST_F(ResolvedChainCacheTest, ResolvedTargetTagSeamLockOrder) { EXPECT_EQ(77, result.load()); } +// Leak-tag identity is (tag, ownership-period stamp): a chain cached during the tag's current +// ownership is answered by the seam and emitted by the dump. +TEST_F(ResolvedChainCacheTest, LeakTagChainOfCurrentOwnerIsResolvedAndEmitted) { + LivenessTracker *lt = LivenessTracker::instance(); + jlong leak = lt->acquireLeakTagForTest(); + ASSERT_TRUE(ReferenceChainTracker::isLeakTag(leak)); + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeLeakEvent(leak, lt->leakTagIncarnation(leak)), + 5, 1000); + EXPECT_EQ(leak, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(leak)); + EXPECT_TRUE(drainContainsTarget((u64)leak)); +} + +// The reviewer's ABA: the chain's object dies (tag released), and the tag is recycled for a new +// object. The old chain must not be attributed to the new owner by the seam nor emitted into the +// dump - and it must not be erased either (the eviction policy owns removal). +TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitted) { + LivenessTracker *lt = LivenessTracker::instance(); + jlong leak = lt->acquireLeakTagForTest(); + ASSERT_TRUE(ReferenceChainTracker::isLeakTag(leak)); + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeLeakEvent(leak, lt->leakTagIncarnation(leak)), + 5, 1000); + ASSERT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + + // Owner dies: released, not yet re-acquired. + lt->releaseLeakTagForTest(leak); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(leak)); + EXPECT_FALSE(drainContainsTarget((u64)leak)); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()) + << "emission must not erase the stale chain"; + + // A new object receives the recycled tag (LIFO). + jlong reused = lt->acquireLeakTagForTest(); + ASSERT_EQ(leak, reused); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(reused)) + << "the dead owner's chain was attributed to the tag's new owner"; + EXPECT_FALSE(drainContainsTarget((u64)reused)); + EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + + // Once the new owner's own chain is cached, it is the one that matches. + ReferenceChainsTestAccessor::cacheChain( + 6, makeLeakEvent(reused, lt->leakTagIncarnation(reused)), 6, 1000); + EXPECT_EQ(reused, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(reused)); + std::vector out; + ReferenceChainsTestAccessor::drain(&out); + ASSERT_EQ(1u, out.size()) << "only the current owner's chain is emitted"; + EXPECT_EQ(lt->leakTagIncarnation(reused), out[0]._leak_tag_incarnation); +} + +// Within one ownership period the leak-tag path is generation-agnostic: a chain an earlier search +// cached for the same owner still answers after a restart. +TEST_F(ResolvedChainCacheTest, LeakTagChainMatchesAcrossGenerationsForSameOwner) { + LivenessTracker *lt = LivenessTracker::instance(); + jlong leak = lt->acquireLeakTagForTest(); + ASSERT_TRUE(ReferenceChainTracker::isLeakTag(leak)); + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeLeakEvent(leak, lt->leakTagIncarnation(leak)), + 5, 1000); + ReferenceChainsTestAccessor::setSearchStartNsForTest(2000); + EXPECT_EQ(leak, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(leak)); + EXPECT_TRUE(drainContainsTarget((u64)leak)); +} + +// Chains whose target is a frontier tag (no leak tag) are not subject to the ownership check: +// emitted regardless of what the leak-tag pool does. +TEST_F(ResolvedChainCacheTest, FrontierTargetChainsAreEmittedRegardlessOfPoolState) { + LivenessTracker *lt = LivenessTracker::instance(); + ReferenceChainsTestAccessor::cacheChain(1, makeEvent(7), 7, 0); + EXPECT_TRUE(drainContainsTarget(7)); + jlong leak = lt->acquireLeakTagForTest(); + ASSERT_TRUE(ReferenceChainTracker::isLeakTag(leak)); + lt->releaseLeakTagForTest(leak); + lt->leakTagPoolResetForTest(); + EXPECT_TRUE(drainContainsTarget(7)); +} + // Pause-time pacing controller: pause-time-SLO feedback loop TEST_F(ReferenceChainsTest, PacingHoldsSteadyWhenPassesLandExactlyOnCeiling) { diff --git a/ddprof-lib/src/test/fuzz/fuzz_referenceChainFrontier.cpp b/ddprof-lib/src/test/fuzz/fuzz_referenceChainFrontier.cpp index 4163c875e6..9f5c9b12ec 100644 --- a/ddprof-lib/src/test/fuzz/fuzz_referenceChainFrontier.cpp +++ b/ddprof-lib/src/test/fuzz/fuzz_referenceChainFrontier.cpp @@ -448,7 +448,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { shadow[idx].leak_tag = leak; } } - table.setLeakTag(tag, leak); + table.setLeakTag(tag, leak, /*leak_tag_incarnation=*/0); break; } case 9: { // LOOKUP - I2 checked by the sweep below diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java index e213b9a005..ed2528c89b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java @@ -15,6 +15,11 @@ public abstract class AbstractProcessProfilerTest { + // Upper bound for joining each output reader thread once the child has exited (or been + // destroyed). The readers normally hit EOF right after the child ends; the bound only stops a + // descendant that inherited the pipes from keeping launch() from returning. + private static final long READER_JOIN_TIMEOUT_MS = 10_000; + public static final class LaunchResult { public final boolean inTime; public final int exitCode; @@ -146,6 +151,14 @@ protected final LaunchResult launch(String target, List jvmArgs, String p.destroyForcibly(); p.waitFor(5, TimeUnit.SECONDS); } + // Process.waitFor() only orders the child's exit, not the readers' last callbacks. Child modes + // that call System.exit(0) skip the [ready] handshake, so nothing else guarantees the final + // marker lines have been handed to the callbacks before the caller asserts on the state they + // set. Join both readers (bounded, on both the in-time and the timeout path): Thread.join() + // makes every callback write visible to the caller. If a reader is still alive after the bound, + // return anyway, as before. + stdoutReader.join(READER_JOIN_TIMEOUT_MS); + stderrReader.join(READER_JOIN_TIMEOUT_MS); return new LaunchResult(val, p.exitValue()); } } \ No newline at end of file diff --git a/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md b/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md new file mode 100644 index 0000000000..36a4730b8f --- /dev/null +++ b/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md @@ -0,0 +1,182 @@ +# Spec: Leak-tag ownership-period identity for cached chains, and launcher reader join + +This spec covers the PR-800 change requests `4282599797` (leak-tag reuse in the cached-chain +lookup) and `4282599807` (join the stdout/stderr readers before `launch()` returns). + +The design is in `.sphinx/address/design.md`. It converged after 2 adversary rounds. Its "Cluster +Rules & Invariants" section (A-R1..A-R10, B-R1..B-R4) is the normative source of truth for this +spec. + +## Problem + +- **Stale chain attributed to a new object (seam).** + `ReferenceChainTracker::resolvedChainTargetTagForTest()` (`referenceChainEvents.cpp:513-531`) + answers a leak-tag query with `object_tag` whenever any cached chain has + `_target_tag == object_tag`. + - Leak tags are recycled. `cleanup_table()` releases the tag when its object dies + (`livenessTracker.cpp:343-345`), and `acquireLeakTag()` hands it out again in LIFO order. + - Cached chains are not invalidated when their target dies. + - So the seam can match a dead object's chain against a new object that now holds the same tag. + - The code comment and the header doc both claim that tag equality is exact object identity. +- **The same mix-up in emitted events (sibling, production).** `drainPendingChainEvents()` + re-emits every cached chain in every dump, including chains whose leak-tagged target has died. + - A consumer joins `ReferenceChain.targetTag` to `HeapLiveObject.leakTag`. That join attributes + the dead object's chain to the tag's new owner. + - `StaticFieldGrowingCollectionScenario.findMatch()` can also match the stale chain. +- **Launcher returns before its readers finish.** `AbstractProcessProfilerTest.launch()` returns + after `p.waitFor()` without joining the stdout/stderr reader threads. Callback side effects, such + as the marker flags the reference-chain tests assert on, can be incomplete or not yet visible + when `launch()` returns. + +## Correct behaviour + +1. **R-STAMP (A-R2).** + - `LivenessTracker` keeps a `u32` ownership-period stamp for every leak-tag slot. + - The stamp changes, under `_leak_tag_pool_lock`, on every ownership transition of the slot: + - `acquireLeakTag()` takes the slot (free → in use); + - `releaseLeakTag()` actually releases it (in use → free); + - `start()`'s ownership rebuild or `leakTagPoolResetForTest()` flips the slot's in-use bit. + - Rejected and out-of-range releases, and rebuilds that leave a slot's ownership unchanged, do + not change the stamp. + - `u32 leakTagIncarnation(jlong tag) const` reads the stamp lock-free with acquire semantics. + It returns 0 for a tag outside the pool range. +2. **R-CAPTURE (A-R3).** + - Every production site that binds a leak tag to a frontier entry stores the current stamp with + it, in `FrontierEntry::leak_tag_incarnation`. The sites are the edge interception in + `heapReferenceCallback`, the root interception in `heapRootCallback`, and + `correlateAdmittedLeakTag`. + - `FrontierTable::setLeakTag` takes the stamp as a required argument with no default. + - `insert()` zeroes the stamp. +3. **R-PROPAGATE (A-R4).** `buildChainEvent()` copies the entry's stamp into a new in-memory field, + `ReferenceChainEvent::_leak_tag_incarnation`. That field defaults to 0 and is never serialized. +4. **R-OWNER (A-R5).** A cached chain describes the current owner of its leak tag if and only if + both of these hold: + - `isLeakTag(_target_tag)`; + - `_leak_tag_incarnation == LivenessTracker::instance()->leakTagIncarnation(_target_tag)`. + + One private helper implements this predicate, and both the seam and emission use it. +5. **R-SEAM (A-R6).** For a leak tag `L`, `resolvedChainTargetTagForTest(L)` returns `L` only if + some cached chain satisfies R-OWNER for `L`. Otherwise it returns 0. + - Search generation does not matter: the same owner's chain from an earlier generation still + matches. + - The frontier-tag path, the `<= 0` guard and the lock order are unchanged. +6. **R-EMIT (A-R7).** `drainPendingChainEvents()` does not emit a cached chain that has a leak-tag + target and fails R-OWNER. + - It also does not erase that chain. The existing eviction policy still decides when it leaves + the cache. + - Every other chain is emitted exactly as before. +7. **R-JOIN (B-R2/B-R3).** On both exit paths of `launch()` (in-time exit, and timeout followed by + forced destroy), the method joins `stdoutReader` and `stderrReader` before it returns. + - Each join is bounded by a named constant (10 s per reader). + - When the bound expires, `launch()` returns as before. + - `LaunchResult` and the `[ready]`/STOP handshake are unchanged. +8. **R-DOC.** Each of these documents its contract as "identity = (leak tag, ownership-period + stamp)" and no longer claims "exact object identity": + - the seam's in-body comment; + - the `referenceChains.h` seam doc; + - the `drainPendingChainEvents()` header doc; + - the `JavaProfiler.getReferenceChainResolvedTargetTagForTest0` Javadoc. + +## Constraints + +- The JFR format and metadata are unchanged. `flightRecorder.cpp` is not touched. +- `sizeof(FrontierEntry)` stays the same. Put the new `u32` directly after `u8 state` and add a + `static_assert` on the size. +- Heap callbacks add exactly one lock-free atomic load per leak-tag interception. They add no lock + and no allocation. +- The emission filter adds only lock-free loads under `_resolved_chains_lock`. +- No new lock-order edges. `_leak_tag_pool_lock` stays a leaf lock. Do not take + `_leak_tag_pool_lock` from reference-chain code. +- `improveChain()` and `reparentToDurableRoot()` must not touch the stamp. The stamp moves together + with `leak_tag`, and only `insert()`/`setLeakTag()` write it. +- Java: no signature changes to `launch(...)`/`LaunchResult`. Formatting goes through spotless. +- C++ style follows the existing two-space indentation and comments of roughly 100 columns. + +## Scope + +### Primary fixes + +- `ddprof-lib/src/main/cpp/referenceChainEvents.cpp:526` (`4282599797`). + - Defect class: identity by a recycled numeric key without an ownership-period check (ABA). + - Fix: + - add the stamp array and accessor in `livenessTracker.{h,cpp}`, bumping the stamp in acquire, + release, `start()`'s rebuild and the test reset; + - add `FrontierEntry::leak_tag_incarnation` and change `setLeakTag`'s signature in + `referenceChainFrontier.{h,cpp}`; + - capture the stamp at the three binding sites (`referenceChainWalk.cpp:299`, + `referenceChainTraversal.cpp:94`, `referenceChainEvents.cpp:782`); + - add the event field `u32 _leak_tag_incarnation` to `ReferenceChainEvent` + (`event.h:136`, initialized to 0 in the constructor) and fill it in `buildChainEvent`; + - add the owner predicate and apply it in the seam; + - update the docs (`referenceChains.h`, `JavaProfiler.java:685-691`). +- `ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java:144` + (`4282599807`). + - Defect class: an asynchronous output consumer is not joined before the result is returned. + - Fix: do a bounded join of both reader threads after the wait/destroy block, on both paths. + +### Auto-expanded sibling fixes + +- `ddprof-lib/src/main/cpp/referenceChainEvents.cpp:792-812`, `drainPendingChainEvents()`. + - Same ABA class, on the production emission path. + - Without this fix, the scenario-level identity assertion the reviewer named still matches the + stale chain. + - Apply the R-OWNER skip without erasing (R-EMIT). + +The following candidates were considered and rejected because they are class signals, not +identity: + +- `leak_tag != 0` checks in `buildDiscoveredInstanceChains` coverage accounting + (`referenceChainEvents.cpp:679-690`); +- anchor-tier checks (`referenceChainAnchors.cpp:161,211`); +- noise eviction (`referenceChainEvents.cpp:742`); +- `tagLeakInstances`' `existing >= LEAK_TAG_BASE` branch, which compares an object with itself. + +Frontier-tag ABA is already guarded by `source_search_ns`. + +No launcher siblings: `waitFor(` appears only in `AbstractProcessProfilerTest`, and every +child-process test goes through that one `launch()`. + +## Test plan (tester owns test edits) + +- `livenessTracker_ut.cpp` `LeakTagPoolTest`: + - the stamp changes on acquire and on release; + - the stamp is unchanged after a rejected double release and after out-of-range releases; + - after release and re-acquire of the same tag (LIFO), the stamp differs from the first + ownership's; + - `leakTagIncarnation()` returns 0 out of range. +- `referenceChainsTrackerTests.inc` `ResolvedChainCacheTest`. Use `makeEvent` with a leak-range + target, set `_leak_tag_incarnation`, and drive the pool through the `LivenessTracker` test + accessors: + - (a) a chain whose stamp equals the current stamp is returned by the seam for `L` and emitted + by `drain`; + - (b) after `acquire` + `release` (+ re-`acquire`) of `L`, the same cached chain is rejected by + the seam (0) and is absent from `drain`, while `resolvedChainCount()` is unchanged (no erase); + - (c) the seam still matches across a search-generation change when the stamp is unchanged; + - (d) chains with a frontier-tag target are emitted regardless of pool state. + + Reset the pool in SetUp/TearDown so the stamp state stays isolated between tests. +- `referenceChainsEventTests.inc:361,367`: update the direct `setLeakTag` calls to pass the current + stamp. `CorrelateAdmittedLeakTagSetsEntryAndDiscovers`: assert that the entry's stamp equals + `leakTagIncarnation(kLeakTag)`. +- Java: the existing `ExternalProcessReferenceChainTest`/`JavaProfilerTest` runs are the regression + check for R-JOIN. No new Java test is needed beyond these runs: the race cannot be forced + deterministically. +- Verification: `./gradlew :ddprof-lib:gtestDebug` (or the per-binary targets for + `livenessTracker_ut` and `referenceChains_ut`), the debug Java test build for the reference-chain + tests, and `spotlessCheck`. + +## Assumptions + +- (≥90%) Lifetime-aware identity, rather than cache invalidation, is the minimal correct fix. An + invalidated chain would be rebuilt from the dead object's frontier entry on the next poll. +- (≥90%) Not emitting chains whose leak-tag owner is gone matches the documented "live sample" + contract of `_resolved_chains`. Such a target has no `HeapLiveObject` to correlate with. +- (≥90%) `releaseLeakTag()` runs only for dead rows (`livenessTracker.cpp:338-346`; there is no + live-row eviction release). So at binding sites (a) and (b), a live object carrying `L` implies + that its row owns `L` now. +- (≥90%) No reference-chain gtest acquires pool tags (only `leakTagPoolResetForTest()` at + `referenceChainsPodTests.inc:42`). So the existing drain and seam expectations hold with stamp 0. +- Residual: the u32 stamp wraps after 2^32 transitions of one slot. This is accepted and + documented at the field. +- Residual: if a pipe-holding descendant outlives the 10 s bound, the reader join gives up and the + original race window remains (best effort). From 615e257a6b471dd1a986c81977844dcee61a69ca Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 16 Oct 2026 10:41:57 +0200 Subject: [PATCH 13/28] Comment cleanup: drop narration and history notes, keep intent Trim comments across the branch to non-obvious intents and whys: remove ticket/plan references, development-history wording, duplicated rationale blocks, restatements of adjacent code, and dangling half-written fragments; fix wrong cross-references. No code changes. --- ddprof-lib/src/main/cpp/livenessTracker.h | 17 +- .../src/main/cpp/referenceChainEvents.cpp | 2 - .../src/main/cpp/referenceChainFrontier.cpp | 13 +- .../src/main/cpp/referenceChainFrontier.h | 4 +- .../src/main/cpp/referenceChainTraversal.cpp | 6 +- .../src/main/cpp/referenceChainWalk.cpp | 2 +- ddprof-lib/src/main/cpp/referenceChains.h | 55 +- .../com/datadoghq/profiler/JavaProfiler.java | 87 +- .../src/test/cpp/livenessTracker_ut.cpp | 13 +- .../test/cpp/referenceChainsAnchorTests.inc | 64 +- .../test/cpp/referenceChainsEventTests.inc | 25 +- .../test/cpp/referenceChainsTestAccessors.h | 126 +-- .../test/cpp/referenceChainsTrackerTests.inc | 145 +--- .../profiler/AbstractProcessProfilerTest.java | 11 +- .../profiler/AbstractProfilerTest.java | 6 +- .../datadoghq/profiler/ExternalLauncher.java | 39 +- .../ExternalProcessReferenceChainTest.java | 98 +-- .../LeakTagCorrelationReferenceChainTest.java | 35 +- .../LeakTagCorrelationScenario.java | 9 +- .../referencechains/LeakingCacheScenario.java | 164 ++-- .../ReferenceChainAssertions.java | 28 +- .../ReferenceChainJfrParserTest.java | 57 +- .../ReferenceChainTestSeamsTest.java | 134 +-- .../ReferenceChainTrackingTest.java | 788 +++++------------- .../StaticFieldGrowingCollectionScenario.java | 77 +- .../ThreadLocalLeakReferenceChainTest.java | 16 +- .../ThreadLocalLeakScenario.java | 41 +- 27 files changed, 612 insertions(+), 1450 deletions(-) diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index d49972e778..de69a8f9e1 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -652,10 +652,8 @@ class alignas(alignof(SpinLock)) LivenessTracker { // --- Per-klass population tracking (cleanup_table()'s epoch-advance pass only) --- // Resolves the StringDictionary id for `ref`'s class (GetObjectClass + - // GetClassSignature + Profiler::lookupClass(), also used by flush_table()) - // - this is the "genuinely new - // cost on an existing pass" the design doc flags, previously paid only at - // JFR-flush time. Returns 0 (StringDictionary's own "no entry" sentinel) + // GetClassSignature + Profiler::lookupClass(), also used by flush_table()). + // Returns 0 (StringDictionary's own "no entry" sentinel) // if the name could not be resolved or interned. u32 resolveKlassId(JNIEnv *env, jobject ref); @@ -781,8 +779,7 @@ class alignas(alignof(SpinLock)) LivenessTracker { // The sustained-trend gate (this class's own header comment above, // "Sustained-trend gate") - both-required growth-magnitude and floor-rise - // tests, design doc's original "mean of thirds" choice since replaced by - // full-window least-squares regression (see ringWindowStats, + // tests, computed by full-window least-squares regression (see ringWindowStats, // livenessTracker.cpp - cheap, allocation-free, one pass over the // ring, no sorting or extra storage). A single scan // (ringWindowStats(), livenessTracker.cpp) both derives the pass/fail @@ -792,8 +789,7 @@ class alignas(alignof(SpinLock)) LivenessTracker { // false (leaving entry.cached_slope untouched) if entry.ring_fill is below // KLASS_POPULATION_MIN_FILL_FOR_TREND - not enough history yet to trust a // trend; callers must check ring_fill themselves before trusting - // cached_slope, exactly as they checked this method's own return value - // before cached_slope existed. + // cached_slope. // // Called from recordKlassPopulationSampleLocked() every time a new sample // is pushed (both the production path, @@ -1411,9 +1407,8 @@ class alignas(alignof(SpinLock)) LivenessTracker { removeKlassPopulationEntryLocked(syn_slot); } } - // Find-or-create the entry under real_id (previously a silent no-op - // when absent - but with aliasing, set-representative-first is the - // load-bearing order: scenarios must establish the alias BEFORE any + // Find-or-create the entry under real_id: with aliasing, + // set-representative-first is the load-bearing order: scenarios must establish the alias BEFORE any // seeding, and with no entry yet there is nothing to store the // representative into. Creating via a count-0, epoch-0 ring sample // reuses recordKlassPopulationSampleLocked()'s own creation branch diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index e55ad2565c..fb2b641c35 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -25,8 +25,6 @@ #include #include -// Target-selection bridging step - LivenessTracker's leak-candidate ranking feeds - void ReferenceChainTracker::requeueChainRootForRotation(jlong tag) { if (_frontier == nullptr || tag <= 0) { return; diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp b/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp index 0cef418bd3..aaaebd5f28 100644 --- a/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.cpp @@ -80,8 +80,7 @@ bool FrontierTable::growLocked(int required_cap) { "ReferenceChains: frontier table resize to %d entries failed", newcap); return false; } - // realloc() does not zero the newly grown region - clear it so lookup() never returns garbage - // state for a slot that hasn't been inserted yet. + // Zero the grown region so lookup() never returns garbage for a not-yet-inserted slot. memset(tmp + _table_cap, 0, sizeof(FrontierEntry) * (newcap - _table_cap)); Counters::increment(REFERENCE_CHAIN_FRONTIER_TABLE_BYTES, (jlong)(newcap - _table_cap) * sizeof(FrontierEntry)); @@ -131,8 +130,6 @@ bool FrontierTable::insert(jlong tag, jlong parent_tag, u32 referrer_klass, while (sz < idx + 1 && !_table_size.compare_exchange_weak(sz, idx + 1, std::memory_order_relaxed)) { - // sz reloaded with the current value by compare_exchange_weak on failure; retry until - // either this thread wins or another thread already advanced _table_size past idx + 1. } return true; } @@ -225,8 +222,6 @@ bool FrontierTable::improveChain(jlong tag, jlong parent_tag, u32 referrer_klass, u32 depth, u8 root_kind, jint referrer_field_index, u8 edge_kind, jlong referrer_class_tag) { - // Replace a shallow root-attached entry (parent_tag == 0, depth == 0) with a deeper - // chain-attached entry when the object is reached via a longer path. if (tag <= 0 || tag - 1 >= (jlong)INT_MAX) { return false; } @@ -290,8 +285,8 @@ bool FrontierTable::reparentToDurableRoot(jlong tag, jlong new_parent_tag, u32 referrer_klass, jint referrer_field_index, u8 edge_kind) { - // See the declaration's own comment (referenceChains.h) for why this exists as a sibling of - // improveChain(): equal-depth depth-1 noise->real re-parenting. + // See the declaration's comment (referenceChainFrontier.h) for why this exists as a sibling of + // improveChain(): equal-depth depth-1 transient->durable re-parenting. if (tag <= 0 || tag - 1 >= (jlong)INT_MAX || new_parent_tag <= 0 || new_parent_tag - 1 >= (jlong)INT_MAX || new_parent_tag == tag) { return false; @@ -340,7 +335,7 @@ bool FrontierTable::reconstructChain(jlong target_tag, u8 root_kind = 0; int hops = 0; // Bounded by maxCapacity(): every tag maps to a distinct slot (this table's "tags/slots are never - // reused" invariant, see the class comment above), so a well-formed parent_tag chain can visit at + // reused" invariant, see _table_size's comment), so a well-formed parent_tag chain can visit at // most maxCapacity() slots before either reaching parent_tag == 0 or repeating a slot. const int hop_bound = maxCapacity(); for (; hops <= hop_bound && tag != 0; hops++) { diff --git a/ddprof-lib/src/main/cpp/referenceChainFrontier.h b/ddprof-lib/src/main/cpp/referenceChainFrontier.h index 4c2ca19964..666a71cb18 100644 --- a/ddprof-lib/src/main/cpp/referenceChainFrontier.h +++ b/ddprof-lib/src/main/cpp/referenceChainFrontier.h @@ -188,7 +188,7 @@ class alignas(alignof(SpinLock)) FrontierTable { } // Same as lookup() above, but assumes the caller already holds the shared lock via - // withSharedLock() below. + // withSharedLock(). bool lookupLocked(jlong tag, FrontierEntry *out) const; // Marks metadata abandoned; the caller must clear the JVMTI tag. @@ -246,7 +246,7 @@ class alignas(alignof(SpinLock)) FrontierTable { std::vector *out_edges = nullptr, FrontierEntry *out_terminal = nullptr); - // Search restart (ReferenceChainTracker::restartSearch(), this class's own header comment): marks + // Search restart (ReferenceChainTracker::restartSearch()): marks // every slot unoccupied again without releasing _table's allocation - a new search's nextTag() // sequence restarts at 1, reusing these same slot indices, so lookup()/insert() must not read // back the previous search's now-irrelevant entries for them. diff --git a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp index f9c1207890..ceb9fb1292 100644 --- a/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainTraversal.cpp @@ -382,7 +382,7 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, ? OS::nanotime() + (u64)_effective_pause_target_ms * 1000000ULL : 0; bool rotation_frontier_cap_hit = false; - // Prong 2 static-anchor descend walks run FIRST inside rotation's slice: they are the + // Static-anchor descend walks run FIRST inside rotation's slice: they are the // highest-value rotation work (bounded, targeted, and the only rotation tier that can reach a // collection-shaped static holder's internals in one pass), and their edges draw down the same // rotation budget the queue-tier batch below uses - a pass whose anchor walks admit the holder's @@ -400,7 +400,7 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni, rotation_edges_admitted += static_anchor_edges_admitted; rotation_budget -= static_anchor_edges_admitted; *truncated = *truncated || static_anchor_truncated; - // B' requeue: resolved-but-unwalked anchors that came from this pass's FIFO drain go back to + // Requeue: resolved-but-unwalked anchors that came from this pass's FIFO drain go back to // the FIFO front, order-preserving, so a pass whose budget died mid-batch walks them first next // pass instead of waiting for the next sweep lap's re-push. if (!static_anchor_unwalked.empty() && @@ -1237,8 +1237,6 @@ bool ReferenceChainTracker::runPass(jvmtiEnv *jvmti, JNIEnv *jni, return err == JVMTI_ERROR_NONE; } -// Pause-time-SLO feedback loop (see this method's declaration in - void ReferenceChainTracker::updatePacing(u64 pass_wall_ns) { // Truncating to whole milliseconds matches every other PidController usage in this codebase // (ObjectSampler/MallocTracer/NativeSocketSampler all feed it integer counts, pidController.h's diff --git a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp index 63f99f4c0e..5acd946065 100644 --- a/ddprof-lib/src/main/cpp/referenceChainWalk.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainWalk.cpp @@ -464,7 +464,7 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback( *tag_ptr, ctx->resolved_chain_tags, ctx->resolved_chain_tag_count); if (was_root_attached_durable) { - // Demotion push (B'): the replaced entry's static/JNI-global attribution was its only + // Demotion push: the replaced entry's static/JNI-global attribution was its only // anchor-tier eligibility, and it is gone now. ctx->tracker->pushAtRiskStaticAnchor( *tag_ptr, ctx->tracker->classTags()->resolve(class_tag)); diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index a24e6f1213..23b6cc2632 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -44,9 +44,8 @@ class ReferenceChainTracker { volatile bool _enabled; // Frontier metadata table. Constructed lazily on the first start() with the flag enabled, sized - // from args._reference_chains_frontier_cap; like LivenessTracker's table (LivenessTracker's own - // table does the same) it survives stop() so it persists across multiple start/stop recording - // cycles. + // from args._reference_chains_frontier_cap; like LivenessTracker's table it survives stop() so it + // persists across multiple start/stop recording cycles. FrontierTable *_frontier; // args._reference_chains_frontier_cap as of the most recent start() call - recorded @@ -215,7 +214,7 @@ class ReferenceChainTracker { // klass_ids from LivenessTracker::topKlassesByGenerationCount() (a faster, un-hysteresis-gated // ranking than the canary candidate set above - see that method's own comment), refreshed once // per BFS-thread tick but only once hasLeakSignal() has already fired via the slower, - // hysteresis-gated selectLeakCandidates() path (per design discussion: this whole mechanism only + // hysteresis-gated selectLeakCandidates() path (this whole mechanism only // cranks once the trend detector has already triggered, so it never needs to wait out that same // hysteresis a second time on its own). u32 _watched_leak_klass_ids[MAX_WATCHED_LEAK_KLASSES]; @@ -336,8 +335,7 @@ class ReferenceChainTracker { // block there and _configured_budget's own comment). bool _urgency_budget_boosted; - // Hysteresis state behind isUrgent(), which used to be a bare `secondsToOOM() < - // OOM_URGENT_THRESHOLD_S` comparison. + // Hysteresis state behind isUrgent(). mutable bool _urgent_latched; mutable int _urgent_release_ticks; mutable bool _urgent_search_spent; @@ -414,7 +412,6 @@ class ReferenceChainTracker { return false; } i = (i + 1) & SLOT_MASK; - // Wrapped around: table full. if (i == start) { return false; } @@ -437,8 +434,7 @@ class ReferenceChainTracker { } } _priority_expand_set; - // B' (find-anchor-holder-eviction / find-anchor-live-feed-design): a static holder richly - // referenced from the running graph is EXCLUDED from the static-anchor tier forever once its + // A static holder richly referenced from the running graph is EXCLUDED from the static-anchor tier forever once its // frontier entry is chain-attached (first admitted via a non-root path, or demoted by // improveChain) - maybeUpgradeRootAttachedRootKind() refuses entries with parent_tag != 0 by // design, so collectStaticFieldAnchorsForRotation() (parent_tag == 0 filter) can never select it. @@ -550,15 +546,10 @@ class ReferenceChainTracker { // GetObjectsWithTags call for the batch + a depth-bounded interface walk per class). static constexpr int ANCHOR_SHAPE_RECONCILE_BUDGET = 128; - // Per-pass cap on how many AT-RISK static holders (frontier entries with parent_tag != 0 - the - // find-anchor-holder-eviction population) drainStaticAnchorFifo() pops for the same - // walkStaticFieldAnchors() batch. + // Per-pass cap on how many AT-RISK static holders (frontier entries with parent_tag != 0) + // drainStaticAnchorFifo() pops for the same walkStaticFieldAnchors() batch. static constexpr int STATIC_ANCHOR_FIFO_DRAIN = 16; - // (Removed: the old single wrapping-index-cursor selection was replaced by tiered selection with - // per-tier cursors — see collectStaticFieldAnchorsForRotation()'s own comment and - // _anchor_container_cursor/_anchor_other_cursor.) - // Cursor over the flattened (slot, tid) enumeration of _candidate_qualifying_tids above, so // walkCandidateThreadLocals()'s THREAD_WALK_MAX_ANCHORS-per-pass cap rotates fairly instead of // always walking the same first candidates' tids. @@ -655,8 +646,6 @@ class ReferenceChainTracker { static constexpr long URGENT_PAUSE_TARGET_MS = 100; // ceiling STW ms per pass static constexpr u64 URGENT_CADENCE_NS = 10000000ULL; // 10ms floor between passes - // True when LivenessTracker::secondsToOOM() projects exhaustion sooner - // Auto-scaled default for _first_pass_budget when Arguments::_reference_chains_first_pass_budget // is unset (0) - see _first_pass_budget's own comment for why plain _budget is the wrong // fallback. @@ -677,7 +666,7 @@ class ReferenceChainTracker { static constexpr u64 MAX_EFFECTIVE_CADENCE_NS = PASS_CADENCE_NS * 4; // 4s // Conversion factor from "edges of budget signal updatePacing()'s clamp could not absorb" to a - // cadence adjustment in nanoseconds - the two are different units (edge count vs. + // cadence adjustment in nanoseconds. static constexpr u64 CADENCE_NS_PER_EDGE_OVERFLOW = 1000000ULL; // 1ms/edge // Budget-borrowing (see _borrowed_budget's own comment): how many consecutive @@ -704,12 +693,9 @@ class ReferenceChainTracker { // Agent-owned BFS thread; startThread()/stopThread() own its lifecycle. pthread_t _thread; - // std::atomic rather than plain volatile bool - volatile alone gives no C++ memory-model - // acquire/release guarantees (it only prevents the compiler from eliding/reordering that one - // variable's own accesses), so a weakly-ordered CPU (e.g. arm64) could let the BFS thread's - // stopThread()- side write (see stopThread()'s own comment) become visible to threadLoop() later - // than intended, missing the shutdown request on one wakeup and sleeping/looping an extra cycle - // before pthread_join() unblocks it. + // Atomic, not volatile: on a weakly-ordered CPU (e.g. arm64) a volatile stopThread() write could + // become visible to threadLoop() late, missing the shutdown request on one wakeup and looping an + // extra cycle before pthread_join() unblocks it. std::atomic _running; // Cooperative-cancellation flag for an in-flight JVMTI FollowReferences walk: stopThread() sets @@ -788,7 +774,7 @@ class ReferenceChainTracker { // see _urgent_latched for the latch/release rules and why the raw comparison flaps. bool isUrgent() const; - // Search restart gate (this class's own header comment): true once _safepoint_pain_budget has + // Search restart gate: true once _safepoint_pain_budget has // drained back to zero (canStartNow()) *and* hasLeakSignal() above reports at least one leak // candidate. bool canAffordNewSearch(u64 now_ns); @@ -896,9 +882,6 @@ class ReferenceChainTracker { const jlong *resolved_chain_tags, int resolved_chain_tag_count); - // Correlate a leak tag with an instance the BFS admitted BEFORE tagLeakInstances() tagged it (its - // JVMTI tag is a frontier tag, its frontier entry has leak_tag == 0). - // One-time retroactive catch-up for a klass_id the moment it FIRST enters _watched_leak_klass_ids // (pollWatchedTargets() calls this only for the newly-added ids in each refresh, never for ones // already being watched). @@ -942,8 +925,8 @@ class ReferenceChainTracker { int *edges_admitted, bool *truncated, bool *frontier_cap_hit, u64 *safepoint_ns); - // Prong 1 of the candidate-scoped reach design (thread-retained taxonomy: ThreadLocal-held caches - // and thread-owned collections): per pass, walk up to THREAD_WALK_MAX_ANCHORS of the current + // Candidate-scoped reach for thread-retained leaks (ThreadLocal-held caches and thread-owned + // collections): per pass, walk up to THREAD_WALK_MAX_ANCHORS of the current // candidates' qualifying tids' live Thread objects (registerThreadObject()'s map above) with // descendFromAnchor() (anchor-gated to ThreadLocalMap, see above). void walkCandidateThreadLocals(jvmtiEnv *jvmti, JNIEnv *jni, int budget, @@ -1127,8 +1110,6 @@ class ReferenceChainTracker { return tag >= LEAK_TAG_BASE && tag < LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE; } - // Max candidates LivenessTracker::selectLeakCandidates() can return. - // Test accessor for _passes_since_last_progress. int passesSinceLastProgressForTest() const { return _passes_since_last_progress; } // Canary-lane backoff state - see _canary_backoff_mult's own comment. @@ -1159,9 +1140,8 @@ class ReferenceChainTracker { void setCandidateFoundBitsForTest(u64 bits) { _candidate_found_bits = bits; } void setSearchStartedForTest(bool v) { _search_started = v; } // OOM-urgency hysteresis state (isUrgent()'s latch/release machine, - // referenceChains.cpp) - the whole point of C5's chaos assertions is this - // state's reaction to a projected-exhaustion sequence, so tests need both - // the drive (isUrgentForTest()) and the raw latch/tick/entitlement reads. + // referenceChains.cpp): tests drive it via isUrgentForTest() and assert on + // the raw latch/tick/entitlement reads. bool isUrgentForTest() const { return isUrgent(); } bool urgentLatchedForTest() const { return _urgent_latched; } int urgentReleaseTicksForTest() const { return _urgent_release_ticks; } @@ -1312,8 +1292,7 @@ class ReferenceChainTracker { // Reports a search's termination state without needing a target tag. bool buildAbandonedEvent(ReferenceChainAbandonedEvent *out) { - // Acquire-load, not a plain relaxed load - see searchState()'s own comment for why: this is the - // same guard-then-read-details pattern. + // Acquire-load: the detail fields below are read only after observing ABANDONED. if (out == nullptr || loadAcquire(_search_state) != SearchState::ABANDONED) { return false; } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index d8c84df2fb..615e9d6959 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -464,12 +464,9 @@ public Map getDebugCounters() { /** * Test seam (debug native builds only): returns the calling thread's profiler tid - * (ProfiledThread::currentTid()'s value on the native side - the same tid space - * {@code seedTidTrendSample0} matches tracked instances' allocating threads - * against). Scenarios seeding per-(klass, tid) trend ramps must call this ON - * the leaking thread and pass the result to {@code seedTidTrendSample0}, or - * the qualifying tid will match no tracked instance and no leak tag will - * ever be assigned to the scenario's real instances. + * (ProfiledThread::currentTid()), the tid space {@code seedTidTrendSample0} matches + * against. Must be called ON the leaking thread, or the seeded tid will match no + * tracked instance and no leak tag will be assigned. */ public static int getTid() { return getTid0(); @@ -545,11 +542,8 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l /** * Test seam (debug native builds only - a no-op returning {@code false}/{@code 0}/an - * empty array in release builds): decouples LivenessTracker's leak-candidate - * detection from ReferenceChainTracker's chain reconstruction, each independently - * verifiable end-to-end without depending on both the probabilistic JVMTI heap - * sampler and the reference-chain BFS search organically producing the right - * conditions in the same test run. + * empty array in release builds): lets LivenessTracker's leak-candidate detection be tested + * without depending on the probabilistic JVMTI heap sampler. *

    * Enables/disables LivenessTracker's per-klass population tracking directly, * bypassing {@code initialize()}'s live-JVM requirement. Returns {@code true} on @@ -562,31 +556,28 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * history for {@code klassId} directly into LivenessTracker's ring buffer, * bypassing real allocation sampling. Repeated calls (with distinct * {@code epoch} values) build up a trend {@link #selectLeakCandidateKlassIds0()} - * can then rank, letting a test assert a slope signal would be generated for a - * chosen klass id without waiting on real GC epochs. + * can then rank. */ public static native void seedKlassPopulationSample0(int klassId, int count, long epoch); /** * Test seam (debug native builds only): seeds one epoch's worth of per-tid trend * history for {@code klassId}, the (klass, tid) qualification - * selectLeakCandidates() now requires on top of the klass-level ramp seeded by + * selectLeakCandidates() requires on top of the klass-level ramp seeded by * {@link #seedKlassPopulationSample0}. Repeated calls with the same rising shape - * (and the same {@code tid}) build the sustained rise that qualifies {@code tid} - * as a leak-site thread. {@code tid} must be the leaking thread's real profiler - * tid from {@link #getTid()} whenever the scenario also relies on leak tagging - * of real tracked instances; see that method's own comment. + * (and the same {@code tid}) qualify {@code tid} as a leak-site thread. {@code tid} + * must come from {@link #getTid()} whenever the scenario relies on leak tagging of + * real tracked instances. */ public static native void seedTidTrendSample0(int klassId, int tid, int count, long epoch); /** * Test seam (debug native builds only): wires {@code representative} in as {@code klassId}'s * leak-candidate representative directly (a fresh weak global ref owned by LivenessTracker), - * bypassing the real allocation-sampling path that would otherwise populate this. Combined - * with {@link #seedKlassPopulationSample0} and {@link #tagAsReferenceChainRoot0}, lets a test - * join a synthetic slope signal to a real, directly-tagged object so - * {@link #pollReferenceChainTargets0()}'s bridging step can be exercised end-to-end with - * neither the real sampler nor the real root-seeded walk involved. + * bypassing the real allocation-sampling path. Combined with + * {@link #seedKlassPopulationSample0} and {@link #tagAsReferenceChainRoot0}, lets + * {@link #pollReferenceChainTargets0()}'s bridging step run without the real sampler or + * root-seeded walk. */ public static native void setKlassPopulationRepresentativeForTest0(int klassId, Object representative); @@ -600,9 +591,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * Test seam (debug native builds only): returns the klass ids LivenessTracker's * real leak-candidate ranking (positive population slope, top 5) currently * selects - the same call ReferenceChainTracker's restart gate and target-polling - * bridge use in production, exposed here so a test can assert a slope signal was - * generated (real or seeded via {@link #seedKlassPopulationSample0}) without - * needing a reference-chain search to also be running. + * bridge use in production. Does not require a reference-chain search to be running. */ public static native int[] selectLeakCandidateKlassIds0(); @@ -610,12 +599,10 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * Test seam (debug native builds only): tags {@code target} and inserts it * directly as a reference-chain frontier root, bypassing ReferenceChainTracker's * normal discovery path (a root-seeded FollowReferences walk) and - * LivenessTracker's leak-candidate selection entirely. Lets a test drive - * {@link #runReferenceChainPass0()}/{@link #pollReferenceChainTargets0()} against - * a known, caller-chosen live object. Returns the assigned frontier tag (matching - * the {@code target_tag} a resulting {@code datadog.ReferenceChain} event - * reports), or {@code 0} on failure (reference chains disabled, or the frontier - * table is at capacity). + * LivenessTracker's leak-candidate selection entirely. Returns the assigned + * frontier tag (matching the {@code target_tag} a resulting + * {@code datadog.ReferenceChain} event reports), or {@code 0} on failure (reference + * chains disabled, or the frontier table is at capacity). */ public static native long tagAsReferenceChainRoot0(Object target); @@ -640,18 +627,15 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * Test seam (debug native builds only): drains and returns the number of * reference-chain events queued by {@link #pollReferenceChainTargets0()} so far * (the same queue {@code Profiler.dump()} drains in production to write - * {@code datadog.ReferenceChain} JFR events) - lets a test assert a chain was - * actually reconstructed without needing a real JFR dump. + * {@code datadog.ReferenceChain} JFR events). */ public static native int drainReferenceChainEventCount0(); /** * Test seam (debug native builds only): resets ReferenceChainTracker's search/frontier state - * back to a brand-new tracker's, releasing any tags a previous search still held. Since the - * tracker is a process-wide singleton, an in-process test that needs its own genuine first - * root-seeded walk (runPass() only re-walks from the roots once per search's whole lifetime) - * calls this at the start of its test body to force one, rather than depending on being the - * first reference-chain test to run in a shared test JVM. + * back to a brand-new tracker's, releasing any tags a previous search still held. The tracker + * is a process-wide singleton and runPass() walks from the roots only once per search, so an + * in-process test needing its own root-seeded walk must call this first. */ public static native void resetReferenceChainSearchForTest0(); @@ -697,17 +681,15 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * Test seam (debug native builds only): seeds one heap-floor-ring sample - the input to * {@code LivenessTracker::secondsToOOM()}'s time-to-OOM projection - directly, bypassing the * real {@code GarbageCollectionFinish} callback. {@code timestampNs} values are only ever - * compared against each other, never against a real wall clock, so a test may use any - * self-consistent, strictly increasing sequence to build an arbitrary rising or flat - * heap-usage-over-time history without waiting on real GCs. + * compared against each other, never against a real wall clock, so any strictly increasing + * sequence is valid. */ public static native void heapFloorRecordForTest0(long usedBytes, long timestampNs); /** * Test seam (debug native builds only): overrides the max-heap-size {@code secondsToOOM()} - * projects against, bypassing the real {@code Runtime.maxMemory()} resolution - so a test can - * exercise the projection deterministically, independent of whatever {@code -Xmx} this JVM's - * own shared, no-{@code forkEvery} fork happens to run with. + * projects against, bypassing the real {@code Runtime.maxMemory()} resolution, so the + * projection does not depend on the test JVM's {@code -Xmx}. */ public static native void setMaxHeapBytesForTest0(long maxHeapBytes); @@ -724,19 +706,18 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * Test seam (debug native builds only): reports whether ReferenceChainTracker's search-restart * gate ({@code canAffordNewSearch()} -> {@code hasLeakSignal()}) would currently allow a * fresh/terminal search to start - in particular, whether {@code secondsToOOM()}'s urgent-OOM - * bypass opens this gate even with zero per-klass leak candidate (confirmable in the same test - * via {@link #selectLeakCandidateKlassIds0()}). Unlike {@link #runReferenceChainPass0()}, which - * calls {@code runPass()} unconditionally, this reads the gate itself without running a pass. + * bypass opens this gate even with zero per-klass leak candidates. Unlike + * {@link #runReferenceChainPass0()}, which calls {@code runPass()} unconditionally, this reads + * the gate without running a pass. */ public static native boolean shouldRunPassForTest0(); /** * Test seam (debug native builds only): the number of BFS passes run for the current/most - * recent reference-chain search. Lets a test note this count before creating an object, then - * wait for it to advance before trusting a match against that object - the only way to be - * certain the match came from a pass whose own {@code expandFrontier()} (and therefore {@code - * collectStaleExpandedEntriesForRotation()}) ran strictly after the object existed, rather - * than from the same pass racing the object's creation. + * recent reference-chain search. Waiting for it to advance past the value read before creating + * an object guarantees a later match came from a pass whose {@code expandFrontier()} (and + * therefore {@code collectStaleExpandedEntriesForRotation()}) ran strictly after the object + * existed, not from a pass racing its creation. */ public static native int referenceChainPassesRunForTest0(); diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp index cb79e49098..c0b0630dff 100644 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp @@ -761,11 +761,10 @@ TEST_F(SelectLeakCandidatesTest, HeapFloorCorroborationLowersRequiredHysteresis) } // --- Per-(klass, tid) qualification gate (TidTrend, livenessTracker.h) --- -// The disjoint-tagged-vs-frontier pod finding: a whole-klass rising -// generation count can come from churn spread across MANY allocating -// threads, each retaining a STABLE handful of instances. A klass with a -// qualifying klass-level trend but NO thread whose own trend qualifies is -// NOT a leak candidate. +// A whole-klass rising generation count can come from churn spread across +// MANY allocating threads, each retaining a STABLE handful of instances. A +// klass with a qualifying klass-level trend but NO thread whose own trend +// qualifies is NOT a leak candidate. TEST_F(SelectLeakCandidatesTest, KlassTrendWithoutQualifyingTidIsNotSelected) { LivenessTracker *tracker = LivenessTracker::instance(); @@ -839,9 +838,7 @@ TEST_F(SelectLeakCandidatesTest, RetainedCountBarQualifiesOneCohortShape) { } // A rising per-tid trend alone (below the retained-count bar) qualifies: -// small leaks grow their age span long before their count clears the bar - -// the hotdog pod's simulated-memory-leak thread (12 tracked [B instances, -// age_count rising 2->3) is exactly this shape. +// small leaks grow their age span long before their count clears the bar. TEST_F(SelectLeakCandidatesTest, RisingTidTrendQualifiesBelowCountBar) { LivenessTracker *tracker = LivenessTracker::instance(); diff --git a/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc b/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc index 691b0f41f4..d39be8edff 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsAnchorTests.inc @@ -79,10 +79,9 @@ TEST_F(ReferenceChainsBfsTest, StaticAnchorRotationWalksRootAttachedStaticHolder tracker->stop(); } -// Round-14 tiered selection: a container-shaped anchor (its own class implements Collection/Map - -// the LEAK_BUFFER wrapper shape) admitted at a LATE index position must leap the queue of ~28k -// other-tier anchors (the round-13 hotdog measurement: admission-order selection put the leak -// holder at position ~12-21k against ~4k walk coverage per search - deterministically unreachable). +// Tiered selection: a container-shaped anchor (its own class implements Collection/Map) admitted +// at a LATE index position must leap the queue of ~28k other-tier anchors - admission-order +// selection put the leak holder at position ~12-21k against ~4k walk coverage per search. TEST_F(ReferenceChainsBfsTest, ContainerAnchorLeapsQueueAcrossLargeIndex) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); @@ -137,10 +136,9 @@ TEST_F(ReferenceChainsBfsTest, ContainerAnchorLeapsQueueAcrossLargeIndex) { tracker->stop(); } -// Round-15 fresh-admission priority: the hotdog wrapper is admitted LATE in a search (its holder -// class sits at sweep index 24627 of 33270, so admission lands at the anchor-index TAIL) - behind -// the whole fair container backlog measured at 1633 entries against 44-75-pass search lifetimes -// (the fair container cursor would reach it at pass ~102+, after the search is dead). +// Fresh-admission priority: a container admitted LATE in a search lands at the anchor-index TAIL, +// behind the whole fair container backlog (measured at 1633 entries against 44-75-pass search +// lifetimes - the fair cursor would reach it at pass ~102+, after the search is dead). TEST_F(ReferenceChainsBfsTest, FreshContainerWalkedBeforeFairBacklog) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); @@ -233,7 +231,7 @@ TEST_F(ReferenceChainsBfsTest, FreshContainerWalkedBeforeFairBacklog) { tracker->stop(); } -// Round-14 tier fairness: the other tier (everything not leak-tagged, not container-shaped) still +// Tier fairness: the other tier (everything not leak-tagged, not container-shaped) still // reaches full coverage across wraps - a 40-anchor tier at a 16 budget covers all 40 in exactly 3 // calls with no duplicates within a call. TEST_F(ReferenceChainsBfsTest, AnchorOtherTierFairCoverageAcrossWraps) { @@ -278,9 +276,9 @@ TEST_F(ReferenceChainsBfsTest, AnchorOtherTierFairCoverageAcrossWraps) { tracker->stop(); } -// Round-13/14 restart hygiene: discovered-instance tags are FRONTIER tags; restartSearch() resets -// the frontier table and _next_tag=1, so any surviving discovered slot either fails -// reconstructChain (observed on-pod: 'buildChainEvent failed ... +// Restart hygiene: discovered-instance tags are FRONTIER tags; restartSearch() resets the frontier +// table and _next_tag=1, so a surviving discovered slot would fail reconstructChain() or resolve +// to whatever new entry reuses its tag. TEST_F(ReferenceChainsBfsTest, RestartSearchClearsDiscoveredInstanceTags) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); @@ -313,9 +311,9 @@ TEST_F(ReferenceChainsBfsTest, RestartSearchClearsDiscoveredInstanceTags) { tracker->stop(); } -// Round-19 (pod 289f8 — three JVMs of "canary search, 0/1 candidates found" while leak-tagged -// instances WERE intercepted and chains re-emitted): the marker->leak-tag design migration never -// updated the found criterion. +// A chain whose target carries a leak tag must mark its canary candidate found - otherwise the +// search reports "0/1 candidates found" while leak-tagged instances ARE intercepted and chains +// re-emitted. TEST_F(ReferenceChainsBfsTest, LeakTagChainMarksCanaryFound) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000")); @@ -354,7 +352,7 @@ TEST_F(ReferenceChainsBfsTest, LeakTagChainMarksCanaryFound) { tracker->stop(); } -// B' push site 1 - DEMOTION TIME (find-anchor-holder-eviction / _static_anchor_fifo): when +// At-risk FIFO (_static_anchor_fifo) push site 1 - DEMOTION TIME: when // improveChain() replaces a root-attached durable (STATIC_FIELD/ JNI_GLOBAL) entry with a deeper // chain-attached path, the entry is leaving the anchor tier's eligible population at exactly that // moment - the push must fire right there. @@ -408,7 +406,7 @@ TEST_F(ReferenceChainsBfsTest, DemotionPushFiresWhenImproveChainEvictsRootAttach tracker->stop(); } -// B' push site 2 - SWEEP TIME: the static sweep's class->field edge onto an already-admitted +// At-risk FIFO push site 2 - SWEEP TIME: the static sweep's class->field edge onto an already-admitted // CHAIN-ATTACHED entry (the admission-order eviction shape: born as a non-root child, never // root-attached at all) must feed the FIFO. TEST_F(ReferenceChainsBfsTest, SweepPushFiresOnStaticEdgeOntoChainAttachedHolder) { @@ -450,10 +448,8 @@ TEST_F(ReferenceChainsBfsTest, SweepPushFiresOnStaticEdgeOntoChainAttachedHolder bool truncated = true; ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated)); - // The rotation phase of the same pass drained the pushed tag into the anchor walk, and the walk - // admitted the holder's child - the push itself left no residue in the FIFO (drained empty) and - // never re-attributed the holder's entry (re-rooting is the documented refusal that motivated - // the FIFO in the first place). + // The same pass's rotation phase drained the pushed tag into the anchor walk, which admitted + // the holder's child; the FIFO is left empty. EXPECT_EQ(0u, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); jlong childTag = tags_ever_assigned[holderChildNode]; ASSERT_GT(childTag, 0) << "the FIFO-drained anchor walk never admitted " @@ -462,8 +458,7 @@ TEST_F(ReferenceChainsBfsTest, SweepPushFiresOnStaticEdgeOntoChainAttachedHolder ASSERT_TRUE(frontier->lookup(childTag, &childEntry)); EXPECT_EQ(105, childEntry.parent_tag); EXPECT_EQ(2u, childEntry.depth); - // The holder's entry is untouched by the push - the B' feed records the at-risk shape, it never - // re-attributes the entry. + // The push records the at-risk shape; it never re-attributes the holder's entry. FrontierEntry entry{}; ASSERT_TRUE(frontier->lookup(105, &entry)); EXPECT_EQ(104, entry.parent_tag); @@ -472,7 +467,7 @@ TEST_F(ReferenceChainsBfsTest, SweepPushFiresOnStaticEdgeOntoChainAttachedHolder tracker->stop(); } -// B' mechanics: a chain-attached holder drained from the at-risk FIFO is descend-walked and +// At-risk FIFO mechanics: a chain-attached holder drained from the at-risk FIFO is descend-walked and // intercepts a leak chunk 3 hops below it - the repair for the population the root-attached // collector demonstrably cannot select (the negative control below). TEST_F(ReferenceChainsBfsTest, AtRiskAnchorFifoDrainAndWalkIntercept) { @@ -515,13 +510,12 @@ TEST_F(ReferenceChainsBfsTest, AtRiskAnchorFifoDrainAndWalkIntercept) { JVMTI_HEAP_REFERENCE_STACK_LOCAL)); // Negative control: the root-attached collector selects NOTHING from a table holding only a - // chain-attached holder and a transient root - the pre-B' behavior that stranded the - // dual-reachable population. + // chain-attached holder and a transient root. std::vector selected = ReferenceChainsTestAccessor::collectStaticFieldAnchorsForRotationForTest(4); ASSERT_TRUE(selected.empty()); - // B': push + drain + walk reaches what the collector cannot. + // Push + drain + walk reaches what the collector cannot. ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(105, 28366); std::vector drained; // 16 = ReferenceChainTracker::STATIC_ANCHOR_FIFO_DRAIN (private), the same per-pass drain cap @@ -558,7 +552,7 @@ TEST_F(ReferenceChainsBfsTest, AtRiskAnchorFifoDrainAndWalkIntercept) { tracker->stop(); } -// B' requeue mechanics: a truncated anchor walk reports exactly the RESOLVED-but-unwalked tags, and +// At-risk FIFO requeue mechanics: a truncated anchor walk reports exactly the RESOLVED-but-unwalked tags, and // requeueStaticAnchorFifoFront() restores them to the FIFO front in order with a consistent // membership set - so an at-risk holder that lost its budget turn keeps it for the next pass // instead of waiting for the next sweep lap. @@ -642,7 +636,7 @@ TEST_F(ReferenceChainsBfsTest, TruncatedAnchorWalkRequeuesUnwalkedFifoTags) { tracker->stop(); } -// a this-field self-edge is REAL in the heap - every java.util.Collections$Synchronized* holder +// A this-field self-edge is REAL in the heap - every java.util.Collections$Synchronized* holder // carries mutex == this - so walking such a holder's own subtree (rotation anchor walk or BFS // descent) re-reports the holder as its own child through that field. TEST_F(ReferenceChainsBfsTest, @@ -696,9 +690,8 @@ TEST_F(ReferenceChainsBfsTest, tracker->stop(); } -// the B' at-risk FIFO sat cap-pinned at 1024 because three classes flooded it (klass 1: 1396 -// pushes, klass 215: 1063, klass 1733: 988+), so the LEAK_BUFFER wrapper's pushes (klass 28366) -// were dropped at the cap check and the lane designed to repair exactly its demotion never carried +// Per-class quota: without it the at-risk FIFO sat cap-pinned at 1024 because three classes +// flooded it, so a leak holder's pushes were dropped at the cap check and the lane never carried // it. TEST_F(ReferenceChainsBfsTest, AtRiskFifoPerClassQuotaKeepsFloodOut) { Arguments args; @@ -723,8 +716,7 @@ TEST_F(ReferenceChainsBfsTest, AtRiskFifoPerClassQuotaKeepsFloodOut) { EXPECT_FALSE(ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest( 1000 + (int)quota)); - // The wrapper's push (a different class) lands despite the flood - exactly the push the pod - // dropped. + // A different class's push lands despite the flood. ReferenceChainsTestAccessor::pushStaticAnchorFifoForTest(2000, 28366); EXPECT_EQ(quota + 1, ReferenceChainsTestAccessor::staticAnchorFifoSizeForTest()); @@ -803,9 +795,7 @@ TEST_F(ReferenceChainsBfsTest, AtRiskFifoPerClassQuotaKeepsFloodOut) { EXPECT_FALSE( ReferenceChainsTestAccessor::staticAnchorFifoContainsForTest(20000)); - // Leave the FIFO drained: this test is the one that saturates it, and the reset() seam above - // now clears it for the next test regardless - but a drained ending also keeps this test - // order-independent even if that seam ever regresses again. + // Leave the saturated FIFO drained so later tests do not depend on the reset() seam alone. std::vector final_drain; ASSERT_EQ(1024, ReferenceChainsTestAccessor::drainStaticAnchorFifoForTest( diff --git a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc index 42af0c17a2..2c41da0ace 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsEventTests.inc @@ -9,13 +9,14 @@ TEST_F(ReferenceChainsBfsTest, HopEdgeLabelsDecodeSpecFieldOrdinals) { ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); ASSERT_FALSE(tracker->start(args)); - // Hierarchy mirroring the spec's own numbering example shape: interface IBase { int p; } 1 own - // field interface ISub extends IBase { int x; } 1 own field class Base { int base_f; } 1 own - // field, no super class Holder extends Base implements ISub { int holder_a; Object leakList; } - // interface ISink { Object CONST_A; Object CONST_B; } Spec ordinal spaces - // (jvmtiHeapReferenceInfoField): Holder (class branch): base = ISub(1) + IBase(1) = 2 - // (transitive interfaces, each once); then the superclass chain root-first: base_f@2; then own - // fields in GetClassFields order: holder_a@3, leakList@4. + // Hierarchy mirroring the spec's numbering example: + // interface IBase { int p; } interface ISub extends IBase { int x; } + // class Base { int base_f; } + // class Holder extends Base implements ISub { int holder_a; Object leakList; } + // interface ISink { Object CONST_A; Object CONST_B; } + // Holder's ordinal space (jvmtiHeapReferenceInfoField): transitive interfaces once each + // (ISub + IBase = 2), then the superclass chain root-first (base_f@2), then own fields in + // GetClassFields order (holder_a@3, leakList@4). void *ibase = (void *)0x5001, *isub = (void *)0x5002, *base = (void *)0x5003, *holder = (void *)0x5004, *isink = (void *)0x5005; addClass(ibase, "Lcom/rc/labels/IBase;"); @@ -194,7 +195,6 @@ TEST_F(ReferenceChainsBfsTest, PriorityExpandCapStopsRotationCollectorPushes) { ASSERT_FALSE(tracker->start(args)); FrontierTable *frontier = tracker->frontierTable(); - // One eligible stale-EXPANDED entry. ASSERT_TRUE(ReferenceChainsTestAccessor::insertFrontierEntry( frontier, 1, 0, 0, FrontierEntryState::EXPANDED, JVMTI_HEAP_REFERENCE_STATIC_FIELD)); @@ -260,8 +260,8 @@ TEST_F(ReferenceChainsBfsTest, StaleRotationPrefersLeakParentsOverBlindLap) { // reparentToDurableRoot: a depth-1 entry first admitted through a transient root (stack/JNI local) // is re-parented to a durable root-attached parent at equal depth - the case improveChain() cannot -// express (it requires a strictly deeper path), and exactly the hotdog shape where the singleton -// collection is a depth-0 static root and its elements depth 1. +// express (it requires a strictly deeper path): a singleton collection as a depth-0 static root +// with its elements at depth 1. TEST_F(ReferenceChainsBfsTest, ReparentToDurableRootSwapsTransientForDurable) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); @@ -305,9 +305,8 @@ TEST_F(ReferenceChainsBfsTest, ReparentToDurableRootSwapsTransientForDurable) { } // recordDiscoveredInstance eviction: noise instances fill discovery slots first-come-first-served, -// but a leak-correlated discovery must evict a noise slot when all are full - without eviction, the -// 8 noise instances observed on-pod permanently blocked every later leak-tagged instance of the -// watched class. +// but a leak-correlated discovery must evict a noise slot when all are full - without eviction, +// noise instances permanently block every later leak-tagged instance of the watched class. TEST_F(ReferenceChainsBfsTest, RecordDiscoveredInstanceEvictsNoiseSlots) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=64")); diff --git a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h index 5445b18eec..2aaad343bd 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h +++ b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h @@ -3,11 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -// Test accessors for the reference-chain tracker, gtest-free by design: -// extracted verbatim from referenceChainsCoreTests.inc so the libFuzzer -// heap-graph target (src/test/fuzz/fuzz_referenceChainHeap.cpp) resets and -// observes the process-wide ReferenceChainTracker singleton through the exact -// same friend-class surface as the gtest suite instead of a diverging copy. +// Test accessors for the reference-chain tracker, gtest-free so the libFuzzer +// heap-graph target (src/test/fuzz/fuzz_referenceChainHeap.cpp) shares the +// gtest suite's friend-class surface instead of a diverging copy. // friend declarations: vmEntry.h (VMTestAccessor), referenceChains.h // (ReferenceChainsTestAccessor). @@ -22,7 +20,6 @@ #include "referenceChains.h" #include "vmEntry.h" -// VMTestAccessor - friend of VM (vmEntry.h), lets tests swap VM::_jvmti for a class VMTestAccessor { public: static jvmtiEnv* getJvmti() { return VM::_jvmti; } @@ -37,7 +34,6 @@ class VMTestAccessor { static void setHotspotVersion(int v) { VM::_hotspot_version = v; } }; -// ReferenceChainsTestAccessor - same pattern as VMTestAccessor above, for the class ReferenceChainsTestAccessor { public: static void reset() { @@ -46,18 +42,14 @@ class ReferenceChainsTestAccessor { t->_frontier = nullptr; t->_class_tags = ClassTagTable(); t->_last_resolved_class_count = 0; - // Without these two, a prior test's fully-swept (or partially-swept) - // admitStaticFieldRoots() state survives in this process-wide singleton and can wrongly - // skip the sweep entirely on this test's first pass if its resolved class count happens to - // match whatever an earlier test last left behind - see resetForRestart()'s identical reset - // of these same fields for the production-restart equivalent of this same contract. + // Otherwise a prior test's admitStaticFieldRoots() sweep state survives in the singleton + // and can skip this test's first sweep if the resolved class count happens to match + // (resetForRestart() resets the same fields in production). t->_last_static_field_class_count = -1; t->_static_field_sweep_cursor = 0; t->_static_field_sweep_cycle_truncated = false; t->_next_tag = 1; - // Shared with LivenessTracker (classTagAllocator.h) - process-wide, not - // per-ReferenceChainTracker-instance, so it needs its own reset seam rather than being a - // plain member write. + // Process-wide and shared with LivenessTracker (classTagAllocator.h), not a member. ClassTagAllocator::resetForTest(); t->_search_started = false; t->_tags_released = true; @@ -70,11 +62,9 @@ class ReferenceChainsTestAccessor { t->_pending_expand.clear(); t->_priority_expand.clear(); t->_priority_expand_set.clear(); - // B' at-risk FIFO + its indexes/counters, and the round-15 fresh lane: production clears - // all of these on every restartSearch()/ resetSearchStateForTest(), but this seam predates - // the FIFO and was never given the clears - until round 16 a test left at-risk entries - // behind and only survived because LATER tests' own runPass()s drained the residue with the - // full Bfs mock. + // At-risk FIFO + its indexes/counters and the fresh lane: production clears these on + // every restartSearch()/resetSearchStateForTest(); left behind, at-risk entries leak + // into later tests. t->_static_anchor_fifo.clear(); t->_static_anchor_fifo_set.clear(); t->_static_anchor_fifo_klass_counts.clear(); @@ -117,19 +107,15 @@ class ReferenceChainsTestAccessor { t->_leak_parent_fanout.clear(); t->_borrowed_budget = 0; t->_consecutive_under_target_passes = 0; - // Adaptive batch + lane state: NOT covered by anything above, and a prior test that drove - // expansion leaves a non-zero EMA, a live batch size, a stale pass deadline, and/or a - // mid-alternation lane toggle behind - all of which silently change the next test's - // expandFrontier() arithmetic (exact-value asserts on batch sizing only pass standalone - // otherwise). + // Adaptive batch + lane state: a prior test that drove expansion leaves a non-zero EMA, + // batch size, pass deadline, or lane toggle behind, silently changing the next test's + // expandFrontier() arithmetic. t->_gotw_ema_call_ns = 0; t->_gotw_batch_size = 0; t->_pass_deadline_ns = 0; t->_expand_lane_prefer_priority = true; } - // Search restart + pain budget (SearchRestartTest below) - same rationale as the pacing - // accessors above: private state a test needs to drive/observe directly. static bool canAffordNewSearch(u64 now_ns) { return ReferenceChainTracker::instance()->canAffordNewSearch(now_ns); } @@ -174,9 +160,7 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->canaryStuckRestartCountForTest(); } - // OOM-urgency hysteresis (C5): drive isUrgent()'s latch/release machine - // and read its raw state - same friend-accessor rationale as the canary - // wrappers above. + // OOM-urgency hysteresis: isUrgent()'s latch/release machine and raw state. static bool isUrgentForTest() { return ReferenceChainTracker::instance()->isUrgentForTest(); } @@ -211,9 +195,6 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->_search_pain_ms; } - // Resolved-chain cache: read-only size peek and a pass-through to the private snapshot - // (drainPendingChainEvents()) and insert (cacheResolvedChain()), for ResolvedChainCacheTest - // below - same rationale as hasResolvedChainForTag()/resolvedChainCount() below. static size_t resolvedChainCount() { return ReferenceChainTracker::instance()->_resolved_chains.size(); } @@ -287,9 +268,6 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->drainPendingChainInvalidations(); } - // Target-selection bridging step: read-only peeks into the resolved-chain cache, for asserting - // exactly which klass a chain was resolved for and the tag it was reconstructed from - see - // PollWatchedTargetsTest below. static bool hasResolvedChainForTag(jlong tag) { ReferenceChainTracker *t = ReferenceChainTracker::instance(); return t->_resolved_chains.find(tag) != t->_resolved_chains.end(); @@ -301,9 +279,7 @@ class ReferenceChainsTestAccessor { return it == t->_resolved_chains.end() ? 0 : it->second.source_tag; } - // Leak-tag correlation (design C): read a frontier entry's stored leak tag, and a pass-through - // to the private buildChainEvent(), for LeakTagInterceptionTest below - same friend-accessor - // rationale as hasResolvedChainForTag() above. + // Frontier entry's stored leak tag, or -1 when the entry does not exist. static jlong frontierLeakTag(jlong tag) { ReferenceChainTracker *t = ReferenceChainTracker::instance(); FrontierEntry entry{}; @@ -328,9 +304,7 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->setCandidateKlassIdForTest(idx, klass_id); } - // Round-19: the leak-tag canary found criterion (pod 289f8 — the chase was structurally - // unresolvable after the marker->leak-tag migration; see buildDiscoveredInstanceChains' own - // comment). + // Leak-tag canary found criterion - see buildDiscoveredInstanceChains(). static u64 candidateFoundBitsForTest() { return ReferenceChainTracker::instance()->_candidate_found_bits; } @@ -347,7 +321,7 @@ class ReferenceChainsTestAccessor { nullptr, nullptr, klass_id, current_search_ns); } - // ---- pod-in-a-jar system harness (design-pod-in-a-jar-harness) ---- + // ---- system harness ---- static u8 searchStateForTest() { return ReferenceChainTracker::instance()->_search_state; } @@ -461,8 +435,6 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->candidateDiscoveredCountForTest(slot); } - // recordDiscoveredInstance()/correlateAdmittedLeakTag() are the production paths for the - // leak-correlation tests below. static void recordDiscoveredInstanceForTest(u32 klass_id, jlong tag, bool leak_correlated) { ReferenceChainTracker::instance()->recordDiscoveredInstance( @@ -493,7 +465,6 @@ class ReferenceChainsTestAccessor { t->_static_anchor_index_tags.empty(); } - // Read back discovered-instance slots (frontier tags recorded by recordDiscoveredInstance). static jlong discoveredTagForTest(int slot, int idx) { return ReferenceChainTracker::instance() ->_candidate_discovered_tags[slot][idx]; @@ -554,12 +525,7 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->_search_start_ns = ns; } - // Pause-time pacing controller: read-only peeks at the controller's derived values, and a - // pass-through to the private updatePacing() itself, for ReferenceChainsPacingTest below - same - // rationale as hasResolvedChainForTag()/resolvedChainCount() above (the target-selection - // bridging step): private state a test needs to drive/ observe directly, exposed via this - // existing friend accessor rather than adding public getters/setters to ReferenceChainTracker - // itself. + // Pause-time pacing controller. static int effectiveBudget() { return ReferenceChainTracker::instance()->_effective_budget; } @@ -574,11 +540,8 @@ class ReferenceChainsTestAccessor { static u64 baselineCadenceNs() { return ReferenceChainTracker::PASS_CADENCE_NS; } - // Test-only seams for PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeiling below, which needs - // to start from a controlled below-ceiling/above- baseline point with a freshly reset - // controller (see that test's own comment for why chaining directly off a prior constant-input - // sequence would leave _pause_pid's integral state mid-recovery from that sequence's windup, - // muddying this method's per-step direction assertions with a transient the test is not about). + // Start the controller from a chosen below-ceiling/above-baseline point with a fresh + // _pause_pid, so a prior sequence's integral windup cannot leak into per-step assertions. static void setEffectiveBudget(int v) { ReferenceChainTracker::instance()->_effective_budget = v; } @@ -591,9 +554,7 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->_pause_pid.reset(); } - // Budget-borrowing (referenceChains.h's _borrowed_budget comment): the configured multiplier - // PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeiling below asserts convergence against, - // instead of hardcoding it a second time in the test itself. + // Budget-borrowing (see _borrowed_budget in referenceChains.h). static int borrowCeilingMultiplier() { return ReferenceChainTracker::BORROW_CEILING_MULTIPLIER; } @@ -602,9 +563,6 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->_borrowed_budget; } - // MaybeRevokeBorrowForRootEnumPass* tests below: drive the borrow state directly into "already - // granted" before exercising the revocation-only seam, and a pass-through to that seam itself - - // same rationale as updatePacing()'s own accessor above. static void setBorrowedBudget(int64_t v) { ReferenceChainTracker::instance()->_borrowed_budget = v; } @@ -622,8 +580,6 @@ class ReferenceChainsTestAccessor { pass_wall_ns); } - // ReleaseSearchTagsFailureTest below: read-only peek at whether the tracker still owes a tag - // release before it can allow a restart - see _tags_released's own comment. static bool tagsReleased() { return ReferenceChainTracker::instance()->_tags_released; } @@ -637,11 +593,9 @@ class ReferenceChainsTestAccessor { v, std::memory_order_relaxed); } - // LifecycleChurn test: read side of the flag above - lets the pass thread - // record that a runPass() overlapped a stopThread-style abort request - // (the request is held until the pass boundary, so a post-pass read is - // conservative: it can only over-count by never, under-count by a pulse - // whose set raced the pass's very last callback batch). + // LifecycleChurn test: read side of the flag above. The request is held + // until the pass boundary, so a post-pass read can only miss a pulse whose + // set raced the pass's last callback batch. static bool abortPassRequested() { return ReferenceChainTracker::instance()->_abort_pass_requested.load( std::memory_order_relaxed); @@ -656,11 +610,6 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->_leak_tags_resolved; } - // ResolveLoadedClassesRescansAfterClassCountShrinksAndPartiallyRegrows below: direct - // pass-through to the private resolveLoadedClasses(), plus a read-only peek at the count it - // stashes - the same rationale as tagsReleased() above (private state/behavior a test needs to - // drive/observe directly, without going through a full runPass()/search lifecycle that - // resolveLoadedClasses() alone does not need). static void resolveLoadedClasses(jvmtiEnv *jvmti, JNIEnv *jni) { ReferenceChainTracker::instance()->resolveLoadedClasses(jvmti, jni); } @@ -669,12 +618,9 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->_last_resolved_class_count; } - // Durability re-verification test seams: direct pass-throughs to the private tie-break/rotation - // methods, plus FrontierTable::insert() itself (also private-by-convention here in the sense - // that production code only ever calls it via admitObject()) so tests can set up a frontier - // entry's exact starting root_kind/state/parent_tag without needing a live JVMTI mock for - // IterateOverReachableObjects/FollowReferences (neither is mocked in this file - see the file - // header's FollowReferences- only mock rationale). + // Durability re-verification seams: insertFrontierEntry() sets up an entry's exact + // root_kind/state/parent_tag without a JVMTI heap-walk mock (production only inserts via + // admitObject()). static bool insertFrontierEntry(FrontierTable *frontier, jlong tag, jlong parent_tag, u32 depth, u8 state, @@ -752,8 +698,7 @@ class ReferenceChainsTestAccessor { : (u8)ReferenceChainTracker::AnchorClassShape::NON_CONTAINER; } - // B' at-risk static-anchor FIFO (see _static_anchor_fifo's declaration comment in - // referenceChains.h). + // At-risk static-anchor FIFO (see _static_anchor_fifo in referenceChains.h). using AtRiskAnchor = ReferenceChainTracker::AtRiskAnchor; static constexpr u32 kAtRiskPerKlassCap = ReferenceChainTracker::STATIC_ANCHOR_ATRISK_PER_KLASS_CAP; @@ -852,21 +797,16 @@ class ReferenceChainsTestAccessor { t->_priority_expand.end()); } - // StaleExpandedRotationSkipsPreexistingQueueEntries below: simulates a tag left in - // _priority_expand by a prior pass's truncated expandFrontier() batch (expandFrontier()'s own - // "leave the batch at the front of the source queue for a later pass to retry" comment) without - // driving a full expandFrontier()/JVMTI round-trip to produce one. + // Simulates a tag left in _priority_expand by a prior pass's truncated expandFrontier() + // batch without a full expandFrontier()/JVMTI round-trip. static void pushPriorityExpand(jlong tag) { ReferenceChainTracker *t = ReferenceChainTracker::instance(); t->_priority_expand.push_back(tag); t->_priority_expand_set.insert(tag); } - // Simulates expandFrontier() having fully drained _priority_expand at the end of a pass (the - // common case: rotation's whole selection fit within that pass's rotation_budget slice) - see - // StaleExpandedRotationStarvesHighTagEntryBehindLowTagPopulation below, which needs this to - // model collectStaleExpandedEntriesForRotation() being called fresh on each of several - // simulated passes, the way runPassManualWalk() actually does it once per real pass. + // Simulates expandFrontier() having fully drained _priority_expand at the end of a pass, so + // collectStaleExpandedEntriesForRotation() can be called fresh per simulated pass. static void clearPriorityExpand() { ReferenceChainTracker *t = ReferenceChainTracker::instance(); t->_priority_expand.clear(); @@ -966,8 +906,6 @@ class ReferenceChainsTestAccessor { return ReferenceChainTracker::instance()->_expand_lane_prefer_priority; } - // Leak-tag pool range base (private static) - same friend-access rationale as the AIMD - // constants above. static jlong leakTagBase() { return ReferenceChainTracker::LEAK_TAG_BASE; } diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc index 6528994dc1..d580e57aaa 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -64,17 +64,15 @@ protected: if (active_fixture->dead_refs.count(ref) > 0) { return nullptr; } - return ref; // identity passthrough - see this fixture's own comment + return ref; // identity passthrough: fake jobjects are not real JNI refs } static void JNICALL mock_DeleteLocalRef(JNIEnv *, jobject) { // no-op: this fixture's fake jobject values are not real JNI refs. } - // pollWatchedTargets()'s diagnostic class-name lookup on the candidate's representative: this - // fixture's fake jobjects carry no real class identity, so a fixed non-null jclass plus a fixed - // signature is all GetObjectClass()/GetClassSignature() need to return for that lookup to - // complete without touching a real JVM. + // Fake jobjects carry no class identity; a fixed jclass and signature let + // pollWatchedTargets()'s diagnostic class-name lookup complete. static jclass JNICALL mock_GetObjectClass(JNIEnv *, jobject) { return (jclass)0xC1A55; } @@ -94,11 +92,9 @@ protected: return JVMTI_ERROR_NONE; } - // Seeds LivenessTracker's real population table with a growing series for `klass_id` (20 - // strictly-increasing samples - satisfies selectLeakCandidates()'s min-fill, growth/floor - // magnitude, and sustained-trend hysteresis requirements, livenessTracker.h; 20 rather than the - // 10-sample minimum fill leaves comfortable margin past the hysteresis threshold rather than - // sitting exactly on its boundary) and points its representative at `rep`. + // Seeds a growing population for `klass_id` with representative `rep`. 20 strictly-increasing + // samples clear selectLeakCandidates()'s min-fill and hysteresis gates with margin rather than + // sitting on the 10-sample boundary. void seedGrowingCandidate(u32 klass_id, jweak rep) { int slot; bool created; @@ -179,8 +175,8 @@ TEST_F(PollWatchedTargetsTest, NoDuplicateOnRepeatPoll) { tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); ASSERT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); - // Klass 1 is still flagged (LivenessTracker's ranking doesn't know an event was already emitted - // for it) - a second, third, ... + // Klass 1 stays flagged (the ranking doesn't know a chain was emitted); repeat polls must not + // add duplicates. tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); tracker->pollWatchedTargets(&mock_jvmti, &mock_jni); EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); @@ -212,12 +208,6 @@ TEST_F(PollWatchedTargetsTest, SkipsCandidateWhoseWeakReferenceDied) { tracker->stop(); } -// A cached chain must not re-emit forever: once the klass's representative stops resolving -// (collected, or LRU-evicted from LivenessTracker's population table - -// klassPopulationSetRepresentativeForTest()'s ref is the stand-in for either), the very next poll -// must prune it from _resolved_chains rather than leaving a dump keep re-emitting a chain for a -// sample that is gone (see _resolved_chains' own comment, referenceChains.h, and -// pollWatchedTargets()'s "candidate died, or was evicted" branch). TEST_F(PollWatchedTargetsTest, ChainPersistsAfterRepresentativeDies) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true")); @@ -251,12 +241,9 @@ TEST_F(PollWatchedTargetsTest, ChainPersistsAfterRepresentativeDies) { tracker->stop(); } -// Recording-boundary hygiene (start()'s comment, referenceChains.cpp): a new recording must not -// inherit the previous one's cached resolved chains or queued abandoned events - both would -// re-emit into the new recording's JFR chunks with StringDictionary ids from the wiped class -// dictionary generation. The test seeds both states, restarts through the real stop()/start() -// path (NOT resetSearchStateForTest(), which is an out-of-band reset that also clears this state -// itself), and asserts both are gone while the resolve path still works in the new recording. +// A new recording must not inherit the previous one's cached chains or queued abandoned events: +// both would carry StringDictionary ids from the wiped class dictionary. Restarts through the real +// stop()/start() path, not resetSearchStateForTest(), which would clear this state itself. TEST_F(PollWatchedTargetsTest, RestartDropsPriorRecordingsChainsAndAbandonedEvents) { Arguments args; ASSERT_FALSE(args.parse("referencechains=true")); @@ -448,9 +435,8 @@ TEST_F(PollWatchedTargetsTest, NoOpWhenGcGenerationsDisabled) { ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); ASSERT_FALSE(tracker->start(args)); - // Overrides this fixture's own SetUp() default - exercises the pollWatchedTargets() guard - // covering LivenessTracker's own _gc_generations gate (population tracking's own gate), not - // just this tracker's own _enabled. + // Overrides SetUp(): exercises pollWatchedTargets()'s LivenessTracker _gc_generations guard, + // not just the tracker's own _enabled. LivenessTracker::instance()->setGcGenerationsForTest(false); int fake_object_storage = 0; @@ -467,8 +453,6 @@ TEST_F(PollWatchedTargetsTest, NoOpWhenGcGenerationsDisabled) { tracker->stop(); } -// Resolved-chain cache (ReferenceChainTracker::cacheResolvedChain()/ - class ResolvedChainCacheTest : public ::testing::Test { protected: void SetUp() override { @@ -640,10 +624,9 @@ TEST_F(ResolvedChainCacheTest, RestartDropsPendingTagsBeforeNumericTagReuse) { EXPECT_EQ(101u, dump[0]._target_tag); } -// A brand-new klass arriving with the cache already at MAX_RESOLVED_CHAINS is dropped (and counted -// via REFERENCE_CHAIN_EVENTS_DROPPED, this codebase's own "dropped-event-without-counter" review -// lens) rather than evicting some other still-live sample's chain - but refreshing a klass that is -// already cached still succeeds even at capacity. +// At MAX_RESOLVED_CHAINS a brand-new klass is dropped and counted in +// REFERENCE_CHAIN_EVENTS_DROPPED rather than evicting a live sample's chain; refreshing an +// already-cached klass still succeeds. TEST_F(ResolvedChainCacheTest, OverflowDropsNewKlassButAllowsRefresh) { const int cap = ReferenceChainsTestAccessor::maxResolvedChains(); long long droppedBefore = Counters::getCounter(REFERENCE_CHAIN_EVENTS_DROPPED); @@ -664,8 +647,7 @@ TEST_F(ResolvedChainCacheTest, OverflowDropsNewKlassButAllowsRefresh) { EXPECT_EQ(droppedBefore + 1, Counters::getCounter(REFERENCE_CHAIN_EVENTS_DROPPED)); EXPECT_FALSE(ReferenceChainsTestAccessor::hasResolvedChainForTag((u32)cap)); - // Refreshing an already-cached klass at capacity must still succeed - it reuses that klass's - // existing slot rather than needing a free one. + // A refresh reuses the klass's existing slot. ReferenceChainsTestAccessor::cacheChain(/*source_tag=*/0, makeEvent(999), /*source_tag=*/999, 0); EXPECT_EQ((size_t)cap, ReferenceChainsTestAccessor::resolvedChainCount()); @@ -878,7 +860,7 @@ TEST_F(ResolvedChainCacheTest, LeakTagChainOfCurrentOwnerIsResolvedAndEmitted) { EXPECT_TRUE(drainContainsTarget((u64)leak)); } -// The reviewer's ABA: the chain's object dies (tag released), and the tag is recycled for a new +// ABA: the chain's object dies (tag released), and the tag is recycled for a new // object. The old chain must not be attributed to the new owner by the seam nor emitted into the // dump - and it must not be erased either (the eviction policy owns removal). TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitted) { @@ -954,9 +936,7 @@ TEST_F(ReferenceChainsTest, PacingHoldsSteadyWhenPassesLandExactlyOnCeiling) { u64 startCadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); ASSERT_EQ(4000, startBudget); // starts pinned at the configured ceiling - // A pass landing exactly on the pause-time target is a zero error every call - the controller - // should never move away from its starting point, regardless of how many such passes are - // observed in a row. + // Passes landing exactly on the target are zero error: the controller must never move. for (int i = 0; i < 10; i++) { ReferenceChainsTestAccessor::updatePacing(5 * 1000000ULL); // 5ms EXPECT_EQ(startBudget, ReferenceChainsTestAccessor::effectiveBudget()); @@ -975,8 +955,7 @@ TEST_F(ReferenceChainsTest, PacingShrinksBudgetAndWidensCadenceWhenOverCeiling) int initialBudget = ReferenceChainsTestAccessor::effectiveBudget(); u64 initialCadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); - // A pass taking 10x the pause-time ceiling, fed repeatedly (a constant input - the plan's own - // "does not oscillate indefinitely" scenario). + // Constant input at 10x the pause-time ceiling: must converge, not oscillate. int lastBudget = initialBudget; u64 lastCadence = initialCadence; for (int i = 0; i < 20; i++) { @@ -992,8 +971,7 @@ TEST_F(ReferenceChainsTest, PacingShrinksBudgetAndWidensCadenceWhenOverCeiling) // Moved in the correct direction... EXPECT_LT(lastBudget, initialBudget); EXPECT_GT(lastCadence, initialCadence); - // ...and converged to a fixed point rather than oscillating: one more identical input produces - // no further change. + // ...and converged: one more identical input produces no further change. ReferenceChainsTestAccessor::updatePacing(50 * 1000000ULL); EXPECT_EQ(lastBudget, ReferenceChainsTestAccessor::effectiveBudget()); EXPECT_EQ(lastCadence, ReferenceChainsTestAccessor::effectiveCadenceNs()); @@ -1007,13 +985,9 @@ TEST_F(ReferenceChainsTest, PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeili ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); ASSERT_FALSE(tracker->start(args)); - // Start from a controlled below-ceiling/above-baseline point (as if an earlier over-ceiling run - // had already shrunk/widened them - see the previous test) with a freshly reset controller, - // rather than chaining directly off a constant-input sequence like the previous test's own: - // _pause_pid's integral state would otherwise still be recovering from that sequence's windup - // for many iterations after switching to a smaller-magnitude error, muddying this test's - // per-step "moves in the correct direction every step" assertions with a transient this test is - // not about. + // Start from a shrunk budget / widened cadence with a freshly reset controller: chaining off an + // over-ceiling sequence would leave _pause_pid's integral windup breaking the per-step + // direction assertions below. ReferenceChainsTestAccessor::setEffectiveBudget(2400); ReferenceChainsTestAccessor::setEffectiveCadenceNs( 2 * ReferenceChainsTestAccessor::baselineCadenceNs()); @@ -1021,8 +995,6 @@ TEST_F(ReferenceChainsTest, PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeili int shrunkBudget = ReferenceChainsTestAccessor::effectiveBudget(); u64 widenedCadence = ReferenceChainsTestAccessor::effectiveCadenceNs(); - // Now feed passes comfortably under the ceiling, repeatedly (a constant input, to check - // convergence rather than oscillation). int lastBudget = shrunkBudget; u64 lastCadence = widenedCadence; for (int i = 0; i < 200; i++) { @@ -1035,10 +1007,8 @@ TEST_F(ReferenceChainsTest, PacingGrowsBudgetBackAndRelaxesCadenceWhenUnderCeili lastCadence = cadence; } - // Moved in the correct direction... and, since 50 identical comfortably-under-target passes is - // well past BORROW_WARMUP_PASSES, past the configured ceiling too - budget-borrowing lets it - // converge at the borrowed ceiling (configured budget * multiplier) instead of stalling at the - // plain configured budget. + // Moved in the correct direction, and past the configured ceiling: 200 under-target passes is + // well past BORROW_WARMUP_PASSES, so it converges at the borrowed ceiling (budget * multiplier). EXPECT_GT(lastBudget, shrunkBudget); EXPECT_EQ(4000 * ReferenceChainsTestAccessor::borrowCeilingMultiplier(), lastBudget); EXPECT_LT(lastCadence, widenedCadence); @@ -1080,9 +1050,8 @@ TEST_F(ReferenceChainsTest, MaybeRevokeBorrowForRootEnumPassRevokesJustPastBound ReferenceChainsTestAccessor::setConsecutiveUnderTargetPasses(5); ReferenceChainsTestAccessor::setEffectiveBudget(1500); // as if borrow had raised the ceiling - // Just past the boundary: no longer comfortably under target, so the grant is revoked - // immediately, including re-clamping _effective_budget down to the plain (non-borrowed) budget - // rather than leaving it borrow-inflated until the next ordinary pass's updatePacing() call. + // Just past the boundary: the grant is revoked and _effective_budget re-clamped immediately, + // not left borrow-inflated until the next updatePacing(). ReferenceChainsTestAccessor::maybeRevokeBorrowForRootEnumPass(6 * 1000000ULL); EXPECT_EQ(0, ReferenceChainsTestAccessor::borrowedBudget()); EXPECT_EQ(0, ReferenceChainsTestAccessor::consecutiveUnderTargetPasses()); @@ -1102,7 +1071,6 @@ TEST(PainBudgetTest, SpendCreatesDebtThatBlocksAnImmediateSecondCall) { PainBudget budget(0.01); // 1% ASSERT_TRUE(budget.canStartNow(1000)); // establishes the drain baseline budget.spend(100); // 100ms of debt - // No time has elapsed since the baseline call above - the debt cannot have drained at all yet. EXPECT_FALSE(budget.canStartNow(1000)); } @@ -1118,12 +1086,9 @@ TEST(PainBudgetTest, ZeroRefillRateNeverClearsDebt) { PainBudget budget(0.0); ASSERT_TRUE(budget.canStartNow(0)); budget.spend(1); - // An enormous elapsed time still drains nothing at a 0 refill rate. EXPECT_FALSE(budget.canStartNow(1000000000000ULL)); } -// Search restart (referenceChains.h's own header comment: gating a - class SearchRestartTest : public ::testing::Test { protected: jvmtiInterface_1_ jvmti_tbl{}; @@ -1153,10 +1118,8 @@ protected: VMTestAccessor::setJvmti(orig_jvmti); LivenessTracker::instance()->klassPopulationResetForTest(); LivenessTracker::instance()->setGcGenerationsForTest(false); - // UrgentOOMProjectionBypassesCandidateGate below sets this - reset it here (TearDown always - // runs, even after a fatal ASSERT_* return) rather than as a trailing statement in that - // test body, so a failed assertion can't leak a stale max-heap value into the next test - // sharing this singleton. + // Set by UrgentOOMProjectionBypassesCandidateGate; reset here so a failed ASSERT_* there + // can't leak a stale max heap into the next test. LivenessTracker::instance()->setMaxHeapBytesForTest(-1); } @@ -1168,8 +1131,8 @@ protected: return JVMTI_ERROR_NONE; } - // ReferenceChainTracker::start() -> autoTuneDefaults() queries this whenever LivenessTracker - // reports a max heap > 0 - which UrgentOOMProjectionBypassesCandidateGate below sets. + // start() -> autoTuneDefaults() queries this when max heap > 0 + // (UrgentOOMProjectionBypassesCandidateGate). static jvmtiError JNICALL mock_GetAvailableProcessors(jvmtiEnv *, jint *nprocs) { *nprocs = 1; @@ -1184,18 +1147,14 @@ protected: return JVMTI_ERROR_NONE; } - // runPassManualWalk()'s root enumeration - never invokes the root callback, same "nothing - // reachable from any root" heap model as mock_FollowReferences() above, so the first pass still - // completes immediately with 0 admitted edges. + // runPassManualWalk()'s root enumeration; same empty-heap model as mock_FollowReferences(). static jvmtiError JNICALL mock_IterateOverReachableObjects( jvmtiEnv *, jvmtiHeapRootCallback, jvmtiStackReferenceCallback, jvmtiObjectReferenceCallback, const void *) { return JVMTI_ERROR_NONE; } - // Same seeding helper as PollWatchedTargetsTest above (20 strictly- increasing samples - - // satisfies selectLeakCandidates()'s min-fill, growth/floor magnitude, and sustained-trend - // hysteresis requirements). + // Same as PollWatchedTargetsTest::seedGrowingCandidate(). void seedGrowingCandidate(u32 klass_id, jweak rep) { int slot; bool created; @@ -1212,10 +1171,8 @@ protected: }; TEST_F(SearchRestartTest, WithoutGenerationsSignalRestartStaysUnconditional) { - // gc_generations off (this fixture's SetUp default): canAffordNewSearch() has no candidate - // signal to gate on at all, so a terminal search is immediately eligible to restart - preserves - // this tracker's pre-restart behavior for a referencechains-without-generations setup (this - // class's own header comment, last paragraph). + // gc_generations off (SetUp default): canAffordNewSearch() has no candidate signal, so a + // terminal search is immediately eligible to restart. Arguments args; ASSERT_FALSE(args.parse("referencechains=true")); ReferenceChainTracker *tracker = ReferenceChainTracker::instance(); @@ -1231,9 +1188,8 @@ TEST_F(SearchRestartTest, WithoutGenerationsSignalRestartStaysUnconditional) { } TEST_F(SearchRestartTest, GenerationsEnabledButNoCandidateBlocksFirstSearch) { - // A brand-new tracker must not pay for the initial whole-heap walk/tagging pass either when - // there is no leak candidate yet - shouldRunPass()'s !_search_started branch now shares - // canAffordNewSearch() with the restart gate below (this class's own header comment). + // No leak candidate yet: even the first whole-heap walk is skipped, because + // shouldRunPass()'s !_search_started branch shares canAffordNewSearch() with the restart gate. LivenessTracker::instance()->setGcGenerationsForTest(true); Arguments args; ASSERT_FALSE(args.parse("referencechains=true")); @@ -1287,8 +1243,7 @@ TEST_F(SearchRestartTest, RestartsOnceACandidateAppearsAndResetsPerSearchState) EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); EXPECT_EQ(0, tracker->passesRun()); // restartSearch() zeroed per-search state - // The next runPass() call takes the "first pass of a search" branch again, exactly like a - // brand-new tracker. + // The next runPass() takes the first-pass branch again. ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); EXPECT_EQ(1, tracker->passesRun()); @@ -1306,24 +1261,18 @@ TEST_F(SearchRestartTest, PainBudgetBlocksARestartUntilItDrains) { int fake_object_storage = 0; seedGrowingCandidate(/*source_tag=*/1, /*rep=*/(jweak)&fake_object_storage); - // First-ever search: called via runPass() directly here, bypassing shouldRunPass()'s - // canAffordNewSearch() gate entirely - the candidate seeded above would satisfy that gate - // anyway (this class's own header comment). + // First-ever search via runPass() directly; the seeded candidate would pass the gate anyway. ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); - // Restart #1: _safepoint_pain_budget has never had anything spent into it yet, so this is - // always immediately affordable regardless of this first search's own cost - the cost a search - // incurs only debits the *next* restart's affordability (restartSearch()'s own spend-then-reset - // order), not its own. + // Restart #1: nothing has been spent into _safepoint_pain_budget yet, so it is immediately + // affordable. ASSERT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1)); ASSERT_EQ(SearchState::RUNNING, tracker->searchState()); ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); ASSERT_EQ(SearchState::COMPLETED, tracker->searchState()); - // Pretend this second search cost 1000ms of safepoint time - a mocked FollowReferences call in - // this fixture takes ~0 real wall-clock time, so this accessor stands in for what a real, - // expensive pass would have accumulated into _search_pain_ms on its own. + // The mocked walk costs ~0ms; stand in for an expensive search's _search_pain_ms. ReferenceChainsTestAccessor::setSearchPainMs(1000); // Restart #2: the terminal gate charges the finished search's OWN 1000ms cost BEFORE checking @@ -1332,7 +1281,6 @@ TEST_F(SearchRestartTest, PainBudgetBlocksARestartUntilItDrains) { EXPECT_FALSE(ReferenceChainsTestAccessor::shouldRunPass(2)); EXPECT_EQ(SearchState::COMPLETED, tracker->searchState()); - // Well past the drain point - the debt has cleared, restart #2 proceeds. EXPECT_TRUE(ReferenceChainsTestAccessor::shouldRunPass(1ULL + 200000000000ULL)); EXPECT_EQ(SearchState::RUNNING, tracker->searchState()); ASSERT_TRUE(tracker->runPass(&mock_jvmti, nullptr)); @@ -1343,9 +1291,8 @@ TEST_F(SearchRestartTest, PainBudgetBlocksARestartUntilItDrains) { // hasLeakSignal()'s OOM_URGENT_THRESHOLD_S fast path (referenceChains.h/.cpp): a heap-wide leak // growing fast enough to project exhaustion sooner than the threshold must start a search -// immediately, without waiting for any klass to clear selectLeakCandidates()'s own per-klass -// ring-fill/hysteresis gate - this is the aggressive-leak gap -// GenerationsEnabledButNoCandidateBlocksFirstSearch above documents for the non-urgent case. +// immediately, without waiting for any klass to clear selectLeakCandidates()'s per-klass +// ring-fill/hysteresis gate. TEST_F(SearchRestartTest, UrgentOOMProjectionBypassesCandidateGate) { LivenessTracker::instance()->setGcGenerationsForTest(true); constexpr u64 SEC_NS = 1000000000ULL; @@ -1370,5 +1317,3 @@ TEST_F(SearchRestartTest, UrgentOOMProjectionBypassesCandidateGate) { tracker->stop(); } -// Durability re-verification (correctness hardening). - diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java index ed2528c89b..b62579f2e1 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java @@ -46,10 +46,8 @@ protected final LaunchResult launch(String target, List jvmArgs, String /** * Same as the 6-arg overload above, but with a caller-supplied wait timeout instead of the - * hardcoded 10 seconds - needed by scenarios that run substantially longer than a plain - * attach/init check, e.g. {@code ExternalProcessReferenceChainTest}'s population-growth loop - * (up to 25 rounds plus a grace period, ~20s+ observed in-process, plus a whole separate - * JVM's own startup/classloading cost on top). + * default 10 seconds, for child scenarios that run substantially longer than an attach/init + * check. */ protected final LaunchResult launch(String target, List jvmArgs, String commands, Map env, long timeoutSeconds, Function onStdoutLine, Function onStderrLine) throws Exception { String javaHome = System.getenv("JAVA_TEST_HOME"); @@ -154,9 +152,8 @@ protected final LaunchResult launch(String target, List jvmArgs, String // Process.waitFor() only orders the child's exit, not the readers' last callbacks. Child modes // that call System.exit(0) skip the [ready] handshake, so nothing else guarantees the final // marker lines have been handed to the callbacks before the caller asserts on the state they - // set. Join both readers (bounded, on both the in-time and the timeout path): Thread.join() - // makes every callback write visible to the caller. If a reader is still alive after the bound, - // return anyway, as before. + // set. Thread.join() makes every callback write visible to the caller; the join is bounded + // and a reader still alive after the bound is left behind. stdoutReader.join(READER_JOIN_TIMEOUT_MS); stderrReader.join(READER_JOIN_TIMEOUT_MS); return new LaunchResult(val, p.exitValue()); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java index a6b83f9b04..a157ae1195 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java @@ -73,10 +73,8 @@ public static double scaledSize(JfrEvent item) { protected JavaProfiler profiler; private Path jfrDump; - // Set at the very start of setupProfiler(), before getProfilerCommand() is - // consulted, so a subclass can branch its command on the current test - // method (e.g. distinct hop/budget/frontier-cap values per @Test) without - // needing separate test classes per configuration. + // Set before getProfilerCommand() is consulted, so a subclass can vary its + // command per @Test method instead of needing one test class per configuration. protected TestInfo testInfo; private Duration cpuInterval; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java index dd18a91a58..313f3ca19d 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java @@ -39,23 +39,17 @@ *

  • leak-cache "<start command>|||<scratch dump path>" - starts the profiler with * the given start command, then runs {@link LeakingCacheScenario} in this process and exits * directly (no readiness/stdin-signal handshake, unlike the other modes above) once it - * prints its one result line. Used by {@code ExternalProcessReferenceChainTest} - * (referencechains package) to prove the reference-chains mechanism end-to-end against a - * genuinely separate JVM, not the in-process dynamic-attach lifecycle every other test in - * this module uses - see that scenario's own class comment for why a *separate* process is - * load-bearing here (the one-shot root-seeded BFS walk is a process-wide, once-ever - * resource).
    • + * prints its one result line. Used by {@code ExternalProcessReferenceChainTest}; a separate + * JVM is required because the one-shot root-seeded BFS walk is a process-wide, once-ever + * resource. *
  • leak-static-field "<start command>|||<scratch dump path>" - same packing and * lifecycle as {@code leak-cache} above, but runs {@link StaticFieldGrowingCollectionScenario} * instead: a {@code static final List} field appended to (never reassigned) after - * its owning class has already been swept once by {@code admitStaticFieldRoots()} - - * mirroring the real leak generator found in the {@code prof-analyzer-hotdog-jb} pod.
    • + * its owning class has already been swept once by {@code admitStaticFieldRoots()}. *
  • leak-correlation "<start command>|||<scratch dump path>" - same packing and * lifecycle as {@code leak-cache} above, but runs {@link LeakTagCorrelationScenario} - * instead: a static-collection leak coexisting with ephemeral stack-local noise of another - * class and a large live filler graph, asserting the emitted chain's targetTag is a - * leak-tag-pool tag matching a HeapLiveObject's leakTag - the chain-to-live-heap-sample - * correlation use case.
    • + * instead, asserting the emitted chain's targetTag is a leak-tag-pool tag matching a + * HeapLiveObject's leakTag. * */ public class ExternalLauncher { @@ -228,10 +222,8 @@ public static void main(String[] args) throws Exception { } } } else if (args[0].equals("leak-cache")) { - // "|||" packed into one args[1] string rather - // than extending AbstractProcessProfilerTest.launch()'s generic 2-arg - // (target, commands) contract with a 3rd argument every other mode would have to - // ignore. + // "|||" packed into args[1] so launch()'s + // (target, commands) contract needs no extra argument. String packed = args.length == 2 ? args[1] : ""; int sep = packed.indexOf("|||"); if (sep < 0) { @@ -241,15 +233,11 @@ public static void main(String[] args) throws Exception { String commands = packed.substring(0, sep); String scratchPath = packed.substring(sep + "|||".length()); JavaProfiler instance = JavaProfiler.getInstance(); - // LeakingCacheScenario.run() itself calls instance.execute(commands), not here - - // it needs to seed its cache fixture *before* starting the profiler (see that - // method's own comment for why). + // run() starts the profiler itself: it must seed its cache fixture first. LeakingCacheScenario.run(instance, commands, Paths.get(scratchPath)); System.out.flush(); - // Deliberately exits here rather than falling through to the shared - // "[ready]" + stdin-signal handshake below: this mode runs to completion in one - // shot (no live back-and-forth with the parent needed) and its own result line - // has already been printed by LeakingCacheScenario.run(). + // Skips the "[ready]" + stdin handshake below: the mode is one-shot and its + // result line has already been printed. System.exit(0); } else if (args[0].equals("leak-static-field")) { // Same "|||" packing as leak-cache above. @@ -295,9 +283,8 @@ public static void main(String[] args) throws Exception { System.exit(0); } else if (args[0].equals("threadlocal-leak-late")) { // Same packing as threadlocal-leak, but the leak thread is - // started only AFTER the recording began - the regression - // shape for Profiler::onThreadStart's registerThreadObject() - // wiring (ThreadLocalLeakScenario.run()'s lateThread comment). + // started only AFTER the recording began, exercising + // Profiler::onThreadStart's registerThreadObject() wiring. String packed = args.length == 2 ? args[1] : ""; int sep = packed.indexOf("|||"); if (sep < 0) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java index 6897334f95..c30bf06472 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -23,30 +23,20 @@ import static org.junit.jupiter.api.Assumptions.assumeFalse; /** - * Genuinely separate-process end-to-end coverage for the reference-chains target-selection - * mechanism, complementing {@link ReferenceChainTrackingTest}'s in-process coverage: runs a real - * "leaking Java app" (an ever-growing, never-evicted {@code HashMap}-backed cache - see - * {@link LeakingCacheScenario}) in a genuinely separate child JVM, launched the same way - * {@code JavaProfilerTest}/{@code JVMAccessTest} already launch child processes, and asserts on - * that child's own reported result rather than reading a JFR file back out of this test's own - * process. + * Separate-process end-to-end coverage for the reference-chains target-selection mechanism, + * complementing {@link ReferenceChainTrackingTest}'s in-process coverage: runs a "leaking Java + * app" (see {@link LeakingCacheScenario}) in a child JVM and asserts on the child's reported + * result. * *

    Why a separate process, not another {@code @Test} in {@code ReferenceChainTrackingTest}: - * an earlier attempt added a second success-path {@code @Test} method there, using this exact - * same {@code HashMap}-based leak shape, sharing that class's in-process {@code AbstractProfilerTest} - * dynamic-attach lifecycle. It failed reliably whenever it ran after - * {@code ReferenceChainTrackingTest}'s own {@code ChainLink} test in the same JVM - not a - * test-ordering bug, but a genuine, non-obvious property of {@code ReferenceChainTracker}: * {@code runPass()} (referenceChains.cpp) performs exactly one root-seeded {@code FollowReferences} * walk per search's *entire lifetime* ({@code _search_started}); every later pass only expands * frontier entries that walk already discovered ({@code expandFrontier()}) - it never * re-examines GC roots or revisits an already-{@code EXPANDED} entry for newly-added children. - * Since {@code ReferenceChainTracker} is a process-wide singleton, only the *first* test to ever - * call {@code runPass()} in a given JVM gets a real root-seeded walk; every subsequent test's own, - * independently-rooted local variables are structurally invisible to the mechanism afterward, no - * matter how they are built. A genuinely separate child JVM per scenario sidesteps this - * entirely: each process gets its own fresh {@code ReferenceChainTracker} singleton, and - * therefore its own guaranteed first-ever root walk. + * Since {@code ReferenceChainTracker} is a process-wide singleton, only the *first* test to call + * {@code runPass()} in a given JVM gets a root-seeded walk; later tests' independently-rooted + * locals are invisible to it. A child JVM per scenario gets its own fresh singleton and + * therefore its own first root walk. */ @Tag("slow") public class ExternalProcessReferenceChainTest extends AbstractProcessProfilerTest { @@ -63,49 +53,36 @@ void shouldReconstructReferrerChainInSeparateProcess() throws Exception { // LeakingCacheScenario.run() (inside the child) creates this file on its own first dump() - // an empty placeholder here would make that first dump() attempt fail confusingly. Files.deleteIfExists(scratchDumpPath); - // "start" requires a "jfr,file=..." clause (found the hard way: JavaProfiler.execute() - // throws "Flight Recorder output file is not specified" without one) even though this - // scenario never reads its content - only the explicit dump() calls - // LeakingCacheScenario.run() makes to scratchDumpPath matter. + // "start" requires a "jfr,file=..." clause (JavaProfiler.execute() throws "Flight Recorder + // output file is not specified" without one) even though its content is never read. Path continuousJfrPath = Files.createTempFile("referencechains-external-process-continuous", ".jfr"); try { - // budget=200000 (up from the in-process test's 4000, found via TEST_LOG instrumentation - // in referenceChains.cpp/livenessTracker.cpp): a genuinely fresh external JVM's - // reachable-from-roots graph (full JUnit/JMC/Gradle-worker classpath, bootstrapped from - // scratch - no benefit from a warm, already-running shared worker JVM) is far larger than - // the in-process test's, and each pass's own JVMTI walk cost scales with cumulative - // frontier size, not with this scenario's own allocation rate - budget=4000 stayed - // truncated=1 at 76000+ admitted edges after 19 real passes and never got close to this - // scenario's own cache before the test's own timeout. _budget is the pacing controller's - // hard ceiling (see updatePacing()'s own comment, referenceChains.cpp) - pausetarget alone - // cannot compensate for a ceiling set too low, only pacing *within* it. - // memory=64:l:1.0 - the explicit 100% live-sample ratio removes the - // default 10% subsampling lottery from the tracked-instance pool - // (see LeakTagCorrelationReferenceChainTest's own comment for the - // full analysis); total volume stays capped by the 256KiB interval - // floor, so the tracking table remains tiny. + // budget=200000 (up from the in-process test's 4000): a cold external JVM's + // reachable-from-roots graph (full JUnit/JMC/Gradle-worker classpath) is far larger than + // the in-process test's, and each pass's JVMTI walk cost scales with cumulative frontier + // size - budget=4000 stayed truncated=1 at 76000+ admitted edges after 19 passes and never + // reached the cache before timeout. _budget is the pacing controller's hard ceiling (see + // updatePacing(), referenceChains.cpp) - pausetarget only paces *within* it. + // memory=64:l:1.0 - the 100% live-sample ratio removes the default 10% subsampling + // lottery from the tracked-instance pool; total volume stays capped by the 256KiB + // interval floor, so the tracking table remains tiny. String startCommand = "start,memory=64:l:1.0,generations=true," + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:pausetarget=60000" + ",jfr,file=" + continuousJfrPath.toAbsolutePath(); - // Packed into one args[1] string - see ExternalLauncher's own "leak-cache" mode comment - // for why (avoids extending AbstractProcessProfilerTest.launch()'s generic (target, - // commands) contract with a 3rd argument every other mode would have to ignore). + // Packed into one args[1] string to avoid extending AbstractProcessProfilerTest.launch()'s + // generic (target, commands) contract with a 3rd argument. String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); - // Propagates this JVM's own ddprof_test.config (set by ProfilerTestPlugin.kt on the - // ddprof-test Test task itself, not inherited by a ProcessBuilder-launched child on its - // own) so LeakingCacheScenario can use the same debug-only seeded-representative fallback - // ReferenceChainTrackingTest already relies on instead of depending purely on real - // allocation-sampling timing. + // ddprof_test.config is set on the Test task by ProfilerTestPlugin.kt and is not inherited + // by a ProcessBuilder-launched child; the scenario needs it for its debug-only seeding. List jvmArgs = Collections.singletonList( "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("leak-cache", jvmArgs, packedCommand, Collections.emptyMap(), - // Generous: a whole separate JVM's startup/classloading cost, on top of the same - // up-to-25-round population-growth loop plus grace period that takes ~20s in-process - // (ReferenceChainTrackingTest's own history). + // Generous: a separate JVM's startup/classloading cost on top of the up-to-25-round + // growth loop plus grace period (~20s in-process). 90, line -> { if (line.startsWith(LeakingCacheScenario.FOUND_MARKER) @@ -137,12 +114,10 @@ void shouldReconstructReferrerChainInSeparateProcess() throws Exception { * Mirrors {@link #shouldReconstructReferrerChainInSeparateProcess()}, but against {@link * StaticFieldGrowingCollectionScenario} instead of {@link LeakingCacheScenario}: a {@code * static final List} field appended to (never reassigned) after its owning class has - * already been swept once by {@code admitStaticFieldRoots()} (referenceChains.cpp) - the exact - * shape found in the {@code prof-analyzer-hotdog-jb} pod's real leak generator, as opposed to - * {@code LeakingCacheScenario}'s local-variable (stack-root) cache. Local, fast reproduction of - * whether {@code collectStaleExpandedEntriesForRotation()}'s rotation mechanism actually - * re-expands an already-{@code EXPANDED} static-field-rooted collection to pick up elements - * added after its one-time sweep. + * already been swept once by {@code admitStaticFieldRoots()} (referenceChains.cpp), as opposed + * to {@code LeakingCacheScenario}'s local-variable (stack-root) cache. Checks that + * {@code collectStaleExpandedEntriesForRotation()} re-expands an already-{@code EXPANDED} + * static-field-rooted collection to pick up elements added after its one-time sweep. */ @Test void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exception { @@ -154,14 +129,7 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce Files.deleteIfExists(scratchDumpPath); Path continuousJfrPath = Files.createTempFile("referencechains-static-field-external-process-continuous", ".jfr"); try { - // See ExternalProcessReferenceChainTest.shouldReconstructReferrerChainInSeparateProcess()'s - // own comment for why budget/pausetarget are raised this far above the in-process test's - // defaults for a genuinely fresh, cold external JVM. - // memory=64:l:1.0 - the explicit 100% live-sample ratio removes the - // default 10% subsampling lottery from the tracked-instance pool - // (see LeakTagCorrelationReferenceChainTest's own comment for the - // full analysis); total volume stays capped by the 256KiB interval - // floor, so the tracking table remains tiny. + // budget/pausetarget/memory: see shouldReconstructReferrerChainInSeparateProcess(). // painbudget=100 - the safepoint pain budget exists to protect a production // app from back-to-back expensive passes, but here it starves the engine // on a loaded box: a cold external JVM's search costs seconds of measured @@ -219,11 +187,7 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce assertNotNull(resultLine.get(), "Child process never printed a recognizable result marker on stdout"); assertEquals( - // "byte[]", not byte[].class.getName()'s "[B": StaticFieldGrowingCollectionScenario - // prints match.chain.get(0).getFullName(), and JMC's IMCType renders array types in - // Java source notation, not JVM signature notation - see - // ReferenceChainAssertions.jmcStyleName()'s own comment for why matching required the - // same distinction. + // "byte[]", not "[B": JMC's IMCType renders array types in Java source notation. StaticFieldGrowingCollectionScenario.FOUND_MARKER + "byte[]", resultLine.get(), "Expected a successfully reconstructed chain to a byte[] chunk appended to the static " diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java index 462a07c481..280ba3a149 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java @@ -60,7 +60,7 @@ void shouldCorrelateLeakChainTargetTagWithLiveObjectLeakTag() throws Exception { * The pod-regime variant of the correlation use case: the same scenario under a * pod-like tight pass window (pausetarget in single-digit ms, as on the production-like * pod whose expand window was ~10ms against a 14-41ms GetObjectsWithTags floor at a - * 242k-entry tag map, ev-leaktag-onpod-round4). Full correlation cannot be the pass + * 242k-entry tag map). Full correlation cannot be the pass * criterion at this window width: the scenario's trend gates were tuned for the loose * window's crawl/GC interleaving, and under tight windows the leak klass's real-fold * trend can flip within a single run - the same knife-edge the pod itself lives on @@ -68,8 +68,8 @@ void shouldCorrelateLeakChainTargetTagWithLiveObjectLeakTag() throws Exception { * race; observed: [correlation-tag-out-of-pool] with zero interceptions). What CAN be * asserted deterministically here is the machinery this window exists to exercise: * the leak-accumulation rotation must engage and select FRONTIER-state (un-expanded - * backlog) holders - the round-4 fix whose absence made that tier select zero on the - * pod while holding 51k known parent candidates. GetObjectsWithTags batch control is + * backlog) holders - without that, the tier selected zero on the pod while holding 51k + * known parent candidates. GetObjectsWithTags batch control is * covered unit-level (gotwWindowNs() widening): a real child JVM's local tag map * (~100k filler) keeps the per-call floor under the nominal window, so the widening * regime cannot be reproduced end-to-end locally without a 242k-entry tag map. @@ -79,11 +79,6 @@ void shouldRotateFrontierLeakHoldersUnderPodLikeTightWindows() throws Exception assertCorrelationFound("pausetarget=10", /*requireFrontierRotation=*/ true); } - /** - * Drives the leak-correlation child JVM and asserts the correlated chain, with the - * referencechains pausetarget given by {@code pauseTargetOption} (everything else held - * fixed across the two callers so the variant differs in exactly the pass-window regime). - */ private void assertCorrelationFound(String pauseTargetOption) throws Exception { assertCorrelationFound(pauseTargetOption, /*requireFrontierRotation=*/ false); } @@ -96,10 +91,9 @@ private void assertCorrelationFound(String pauseTargetOption) throws Exception { * leak-accumulation rotation's FRONTIER-state selection is additionally asserted, and * the correlation result markers that the pod regime legitimately produces without the * full correlation completing (not-found, tag-out-of-pool - see the tight variant's own - * comment) are accepted instead of failed. That FRONTIER signal is a level-2 - * {@code TEST_LOG} line, so this variant runs only on DEBUG-instrumented native builds and - * forces reference-chains debug level 2 in the child; the loose variant does not depend on - * logging and runs on every build. + * comment) are accepted instead of failed. This variant runs only on DEBUG-instrumented + * native builds and forces reference-chains debug level 2 in the child; the loose variant + * runs on every build. */ private void assertCorrelationFound(String pauseTargetOption, boolean requireFrontierRotation) throws Exception { @@ -127,17 +121,12 @@ private void assertCorrelationFound(String pauseTargetOption, // Same raised budget as ExternalProcessReferenceChainTest's own scenarios: a genuinely // fresh, cold external JVM's reachable graph is far larger than the in-process tests', // and this scenario adds ~100k filler objects on top. The pausetarget varies by caller. - // memory=64:l:1.0 - the explicit liveness-sample ratio is load-bearing, - // not decoration: without a ratio segment, the default is 10% - // (arguments.h's _live_samples_ratio) and LivenessTracker::track() - // drops 90% of tracked instances probabilistically. This scenario's - // leak cohort is only tens of 1MB chunks, so the default makes the - // (klass, tid) population a per-run Bernoulli lottery - observed live - // as the intermittent "tagLeakInstances ZERO" runs where zero chunks - // survived the filter (passing runs' stable "tagged=5" is exactly the - // ~10% tail of ~50 chunks). 1.0 makes the scenario's documented - // assumption ("every allocation is sampled, tracked, and taggable", - // LeakTagCorrelationScenario) actually true. + // memory=64:l:1.0 - the explicit liveness-sample ratio is load-bearing: the default + // (10%, arguments.h's _live_samples_ratio) makes LivenessTracker::track() drop 90% of + // tracked instances, and with only tens of 1MB leak chunks the (klass, tid) population + // becomes a per-run lottery (observed as intermittent "tagLeakInstances ZERO" runs). + // 1.0 makes LeakTagCorrelationScenario's "every allocation is sampled, tracked, and + // taggable" assumption true. String startCommand = "start,memory=64:l:1.0,generations=true," + "referencechains=true:hops=64:budget=200000:ttl=120000:framecap=2000000:" + pauseTargetOption diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index 93a0c3071d..936f028fd7 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -71,9 +71,7 @@ private LeakTagCorrelationScenario() {} * discovering contents added after a resize is exactly the hard production case * this scenario must prove works: the live holder's re-walk has to admit each new * backing array, and only then can its new chunks be admitted and leak-tag - * intercepted. (An earlier fixed-array version of this fixture sidestepped that - * requirement and still failed - see the scenario history in the investigation - * notes - which is what exposed the rotation starvation this fixture now encodes.) + * intercepted. */ static final List LEAK_SINK = new ArrayList<>(); @@ -245,8 +243,6 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); - // Declared at method scope: the per-round and final-attempt maintenance - // seeding below (in their own debugBuild blocks) reuse these tids. int leakTid = 0; int noiseTid = 0; if (debugBuild) { @@ -290,8 +286,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); JavaProfiler.seedTidTrendSample0(NOISE_KLASS_ID, noiseTid, epoch * 3, epoch); } - // Maintenance continues the seeded ramp (next epoch = SEED_EPOCHS_FOR_HYSTERESIS + 1) - // instead of restarting at 1 - same as StaticFieldGrowingCollectionScenario. + // Maintenance continues the seeded ramp instead of restarting at 1. hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; // Wait for the BFS thread to fully complete its first pass (static sweep AND frontier // expansion) before any growth round - same load-bearing wait and rationale as diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java index ed47fe10f3..3f7eec2b04 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java @@ -18,66 +18,42 @@ import java.util.Map; /** - * The actual "leaking Java app" body run inside a genuinely separate child JVM by - * {@code ExternalLauncher}'s {@code leak-cache} mode, driven by - * {@code ExternalProcessReferenceChainTest}. Deliberately not a JUnit test itself - no - * {@code AbstractProfilerTest}/dynamic-attach lifecycle, no shared-JVM {@code ReferenceChainTracker} - * singleton with any other test - this class's whole point is to run alone in its own process, so - * the one-shot root-seeded BFS walk (see this class's own comment on why a previous attempt at - * folding this scenario into {@code ReferenceChainTrackingTest} as a second {@code @Test} method - * failed) always belongs to this scenario and only this scenario. + * The "leaking Java app" body run inside a separate child JVM by {@code ExternalLauncher}'s + * {@code leak-cache} mode, driven by {@code ExternalProcessReferenceChainTest}. Deliberately not a + * JUnit test: it must run alone in its own process so the process-wide {@code ReferenceChainTracker} + * singleton's one-shot root-seeded BFS walk belongs to this scenario only (see + * {@code ExternalProcessReferenceChainTest}'s class comment). The leak shape is an ever-growing, + * never-evicted {@code HashMap}-backed cache. * - *

    Same realistic-leak shape as {@code ReferenceChainTrackingTest}'s own (now-removed) in-process - * attempt: an ever-growing, never-evicted {@code HashMap}-backed cache, held by a local variable. - * - *

    Why {@code cache} is seeded and populated before the profiler is even started - * (found the hard way, against a real separate-process run): {@code threadLoop()} - * (referenceChains.cpp) calls {@code OS::sleep(_effective_cadence_ns)} - 1 second, initially - - * *before* its very first {@code shouldRunPass()} check, and {@code shouldRunPass()} always - * returns {@code true} on that first-ever check ({@code !_search_started}). So the process's - * one-and-only root-seeded walk fires roughly one second after {@code startThread()}, or earlier - * if woken by a GC-finish signal - a race {@code ReferenceChainTrackingTest}'s own - * {@code shouldReportAbandonedSearchOnTinyFrontierCap()} comment already documents ("that wakeup - * can race the thread's own startup ... and be missed"). An initial in-process attempt at this - * exact scenario created {@code cache} and started allocating only *after* starting the profiler - * (mirroring {@code ReferenceChainTrackingTest}'s {@code gcRootHolder}, which happened to work by - * luck of scheduling in a warm, already-running JVM) - against a genuinely fresh, cold JVM this - * lost the race essentially every time: the one-shot walk caught {@code cache} still empty, - * permanently blocking discovery of everything added to it afterward (a frontier entry once - * marked {@code EXPANDED} is never revisited for new children - see {@code expandFrontier()}'s - * own comment). Seeding {@code cache} with real data *before* {@code profiler.execute()} is even - * called removes the race entirely: no matter how fast the walk fires, {@code cache} is never - * empty when it does. + *

    Why {@code cache} is populated before the profiler is started: + * {@code threadLoop()} (referenceChains.cpp) sleeps {@code _effective_cadence_ns} - 1 second, + * initially - before its first {@code shouldRunPass()} check, which always returns {@code true} + * on that first check ({@code !_search_started}). So the one-and-only root-seeded walk fires + * roughly one second after {@code startThread()}, or earlier if woken by a GC-finish signal. On a + * cold JVM, allocating only after starting the profiler lost that race essentially every time: + * the walk caught {@code cache} still empty, permanently blocking discovery of everything added + * afterward (a frontier entry once marked {@code EXPANDED} is never revisited for new children - + * see {@code expandFrontier()}). Seeding before {@code profiler.execute()} removes the race. */ public final class LeakingCacheScenario { private LeakingCacheScenario() {} - // Arbitrary, scenario-chosen klass id - this process's own LivenessTracker population table - // treats it as an opaque key (see KlassPopulationEntry's own comment) and this process never - // runs any other reference-chains scenario, so no collision risk with e.g. - // ReferenceChainTrackingTest's/ReferenceChainTestSeamsTest's own klass ids in their own, - // separate JVMs. + // Arbitrary klass id - LivenessTracker treats it as an opaque key (see KlassPopulationEntry) + // and this process runs no other reference-chains scenario, so it cannot collide. private static final int CACHED_PAYLOAD_TEST_KLASS_ID = 987301; - // See ReferenceChainTrackingTest.SEED_EPOCHS_FOR_HYSTERESIS's own comment for the full - // derivation: LivenessTracker's hysteresis gate (livenessTracker.h's - // KLASS_POPULATION_MIN_FILL_FOR_TREND=10 / LEAK_TREND_HYSTERESIS_BASE=5) only starts - // incrementing consecutive_positive once ring_fill has reached 10, so a batch of exactly 10 - // seedKlassPopulationSample0() calls caps consecutive_positive at 1 - far short of what - // selectLeakCandidates() (livenessTracker.cpp) requires before ReferenceChainTracker:: - // hasLeakSignal() reports a candidate, which is what this scenario's own generations=true - // startCommand needs to authorize the BFS thread's first real pass at all. + // LivenessTracker's hysteresis gate (livenessTracker.h's KLASS_POPULATION_MIN_FILL_FOR_TREND=10 / + // LEAK_TREND_HYSTERESIS_BASE=5) only starts incrementing consecutive_positive once ring_fill + // reaches 10, so exactly 10 seedKlassPopulationSample0() calls cap consecutive_positive at 1 - + // short of what selectLeakCandidates() requires before hasLeakSignal() reports a candidate, which + // generations=true needs to authorize the BFS thread's first real pass. private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; /** * Durable holder for the cache fixture. The discovered-chain gate suppresses * chains shallower than the first real holder hop that are rooted at a TRANSIENT - * root (stack/JNI local) - a frame-held "leak" is by definition not a leak, so the - * cache must be retained through a static field (the real singleton-collection - * shape) rather than only {@code run()}'s own local, which is exactly the - * transient-rooted shape the gate suppresses. {@code run()}'s local {@code cache} - * variable below aliases this static; the durable root is what makes the emitted - * chain durable-rooted. + * root (stack/JNI local) - a frame-held "leak" is not a leak - so the cache must + * be retained through a static field rather than only {@code run()}'s local. */ private static final Map CACHE = new HashMap<>(); @@ -88,35 +64,25 @@ private LeakingCacheScenario() {} public static final String NOT_FOUND_MARKER = "[chain-not-found]"; /** - * Printed to stdout (with the offending class name appended) if a match was found but its - * chain did not pass through {@code java.util.HashMap}'s own internal storage - see - * {@code ExternalProcessReferenceChainTest} for why this is asserted in the parent process - * rather than here (keeping this scenario's own stdout contract to "found/not-found" and - * leaving richer chain-shape assertions to the JUnit side, the same separation of concerns - * {@code ReferenceChainJfrParserTest} already uses for the lower-level JFR-format proof). + * Printed to stdout (with the offending chain appended) if a match was found but its + * chain did not pass through {@code java.util.HashMap}'s internal storage. */ public static final String NO_HASHMAP_INTERNALS_MARKER = "[chain-found-no-hashmap-internals] "; /** * Printed to stdout (with an "N/M" re-emitted-dump count appended) if the chain was - * successfully reconstructed but did not re-emit into one or more subsequent, - * independently written JFR dumps - i.e. the "snapshot-and-keep" contract of - * {@code ReferenceChainTracker::drainPendingChainEvents()} (referenceChains.cpp), which - * {@code Profiler::dump()} invokes on every dump without clearing the resolved-chain cache, - * failed to hold across dumps. Asserted by {@code ExternalProcessReferenceChainTest}. + * reconstructed but did not re-emit into one or more subsequent JFR dumps - i.e. the + * "snapshot-and-keep" contract of {@code ReferenceChainTracker::drainPendingChainEvents()} + * (referenceChains.cpp) failed to hold across dumps. */ public static final String CHAIN_NOT_PERSISTED_MARKER = "[chain-not-persisted] "; /** - * Seeds {@code cache} with an initial batch, *then* starts the profiler (see this class's own - * comment for why that order is load-bearing), then runs the same population-growth loop - * {@code ReferenceChainTrackingTest} pioneered - dumping to {@code scratchDumpPath} after every - * round (and a short grace period beyond the last round) until a {@code datadog.ReferenceChain} - * event whose {@code chain[0]} is {@link CachedPayload} appears - then prints exactly one - * result line to stdout. Called from {@code ExternalLauncher} (a different package - hence - * {@code public}), which owns the generic child-process bootstrap (JVMTI/library init, process - * exit) this scenario deliberately stays agnostic of; starting the profiler itself happens - * here, not in {@code ExternalLauncher}, specifically so {@code cache} can be seeded first. + * Seeds {@code cache}, *then* starts the profiler (see the class comment for why that order is + * load-bearing), then grows the cache, dumping to {@code scratchDumpPath} after every round + * until a {@code datadog.ReferenceChain} event whose {@code chain[0]} is {@link CachedPayload} + * appears, and prints exactly one result line to stdout. The profiler is started here rather + * than in {@code ExternalLauncher} so {@code cache} can be seeded first. */ public static void run(JavaProfiler profiler, String startCommand, Path scratchDumpPath) throws Exception { Map cache = CACHE; @@ -148,28 +114,18 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD match = findMatch(scratchDumpPath); if (match == null && debugBuild) { - // Same debug-only, seeded-representative short-circuit as ReferenceChainTrackingTest's - // own shouldReconstructReferrerChainThroughUnboundedCacheLeak() (see that method's own - // comment, and SEED_EPOCHS_FOR_HYSTERESIS's own comment above): decouples this scenario's - // assertion from whether the real, probabilistic allocation-sampling-driven slope - // detection happens to notice CachedPayload's own population trend within this scenario's - // fixed round/timeout budget - by clearing hasLeakSignal()'s own hysteresis gate outright, - // which is what actually authorizes the BFS thread's first real pass, rather than assuming - // some earlier pass has already tagged anything. "seed-0" (from seedInitialBatch()) is - // reachable via cache for the scenario's entire lifetime, so it is a safe, always-valid - // representative regardless of which round this fires on. - // Representative BEFORE seeding: setKlassPopulationRepresentativeForTest0 - // resolves the representative's real klass id and aliases the synthetic id - // to it, so the hysteresis seeds below land in the real population entry - // (candidate matching is real-id keyed since the leak-tag pool redesign). + // Debug-only seeded short-circuit: decouples the assertion from whether probabilistic + // allocation sampling notices CachedPayload's trend within the fixed round budget, by + // clearing hasLeakSignal()'s hysteresis gate outright. "seed-0" stays reachable via + // cache for the whole run, so it is a valid representative on any round. + // Representative BEFORE seeding: setKlassPopulationRepresentativeForTest0 aliases the + // synthetic id to the representative's real klass id, so the seeds below land in the + // real population entry (candidate matching is keyed by real id). JavaProfiler.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get("seed-0")); if (!seededTestKlassTrend) { - // Per-(klass, tid) qualification seeds: the qualifying tid is THIS - // thread's - the creator of "seed-0", the always-reachable representative - // the seeded ramp and the match below hang off. (The per-round allocator - // threads are one-shot, so no single one of their tids carries a sustained - // per-tid signal - exactly the one-cohort-per-thread shape the retained- - // count bar / synthetic seeds exist to keep qualifying.) + // The qualifying tid is THIS thread's - the creator of "seed-0". The per-round + // allocator threads are one-shot, so none of their tids carries a sustained + // per-tid signal. int leakTid = JavaProfiler.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { JavaProfiler.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); @@ -217,14 +173,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD return; } - // Across-dumps persistence: a resolved chain must re-emit into EVERY subsequent dump the - // sample survives into, not only the one dump that first observed it. Profiler::dump() - // re-snapshots the whole resolved-chain cache without clearing it on every call - // (drainPendingChainEvents(), referenceChains.cpp - "snapshot-and-keep, not a drain"), so a - // still-live sample's chain re-emits into every chunk. `cache` still holds every - // CachedPayload here, so the sample stays live across these extra dumps; each dump targets a - // brand-new file, so a chain found there can only be present because it was re-emitted into - // that dump, not left over from the round-loop's own scratchDumpPath. + // Across-dumps persistence: Profiler::dump() re-snapshots the resolved-chain cache without + // clearing it (drainPendingChainEvents(), "snapshot-and-keep, not a drain"), so a still-live + // sample's chain must re-emit into every dump. `cache` keeps the sample live; each dump + // targets a brand-new file, so a match there can only come from re-emission. int persistChecks = 3; int persisted = 0; for (int i = 1; i <= persistChecks; i++) { @@ -250,13 +202,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD System.out.println(FOUND_MARKER + match.chain.get(0).getFullName()); } - /** - * Populates {@code cache} with {@code count} entries under a key namespace ("seed-") disjoint - * from the round loop's own ("leak-") - called before the profiler (and therefore - * {@code ReferenceChainTracker}'s BFS thread) even starts, so the process's one-shot - * root-seeded walk can never catch {@code cache} empty (see this class's own comment on why - * that race is otherwise real). - */ + /** Key namespace "seed-" is disjoint from the round loop's "leak-". */ private static void seedInitialBatch(Map cache, int count) { for (int i = 0; i < count; i++) { String key = "seed-" + i; @@ -264,7 +210,6 @@ private static void seedInitialBatch(Map cache, int count } } - /** Temporary diagnostic: print all native debug counters. */ private static void debugDumpCounters(JavaProfiler profiler) { try { Map counters = profiler.getDebugCounters(); @@ -278,7 +223,6 @@ private static void debugDumpCounters(JavaProfiler profiler) { } } - /** Temporary diagnostic: print every datadog.ReferenceChain event's chain[0] class, if any. */ private static void debugDumpAllChainLeaves(Path scratchDumpPath) { try { if (!Files.exists(scratchDumpPath)) { @@ -334,12 +278,10 @@ private static ReferenceChainAssertions.ChainMatch findMatch(Path scratchDumpPat } /** - * Realistic-leak fixture - identical shape to {@code ReferenceChainTrackingTest}'s own - * (now-removed) in-process attempt. The 32 {@code long} fields exist purely so a given number - * of megabytes needs far fewer entries to reach {@code ObjectSampler}'s real 256KiB sampling - * floor - deliberately plain fields, not a nested array (a same-instance companion array would - * be a *second*, separate heap allocation the size-weighted allocation sampler would compete - * for, taking sampling attention away from {@code CachedPayload} itself). + * Leak fixture. The 32 {@code long} fields exist so far fewer entries reach + * {@code ObjectSampler}'s 256KiB sampling floor - plain fields, not a nested array, because a + * companion array would be a second allocation competing for the size-weighted sampler's + * attention. */ static final class CachedPayload { final String key; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java index a8df2e4b0b..e77f1594a9 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java @@ -21,18 +21,12 @@ import java.util.Map; /** - * Shared {@code datadog.ReferenceChain} JFR-parsing helpers, extracted out of - * {@code ReferenceChainTrackingTest} so both that in-process JUnit test and - * {@link LeakingCacheScenario} (run inside a genuinely separate child JVM by - * {@code ExternalProcessReferenceChainTest}) can reuse the exact same JMC-accessor logic - * rather than maintaining two copies. + * Shared {@code datadog.ReferenceChain} JFR-parsing helpers. * - *

    {@link #findMatchForClass(JfrEvents, Class)} is a separate, jafar-backed counterpart to - * {@link #findMatchForClass(IItemCollection, Class)} for {@code ReferenceChainTrackingTest}, which - * loads recordings via {@code AbstractProfilerTest}'s {@code JfrEvents}-returning helpers. - * {@link LeakingCacheScenario} and {@code ReferenceChainJfrParserTest} - * load recordings directly via JMC's {@code JfrLoaderToolkit} and keep using the - * {@code IItemCollection} overload unchanged. + *

    Two overloads because callers load recordings differently: + * {@code ReferenceChainTrackingTest} uses {@code AbstractProfilerTest}'s {@code JfrEvents} + * helpers ({@link #findMatchForClass(JfrEvents, Class)}), while {@link LeakingCacheScenario} loads + * via JMC's {@code JfrLoaderToolkit} ({@link #findMatchForClass(IItemCollection, Class)}). */ public final class ReferenceChainAssertions { private ReferenceChainAssertions() {} @@ -155,15 +149,9 @@ public static JfrChainMatch findMatchForClass(JfrEvents events, Class targetC } /** - * {@code targetClass}'s name in the same format {@code IMCType.getFullName()} uses for array - * types - found the hard way: {@code Class.getName()} renders {@code byte[].class} as {@code - * "[B"} (JVM internal signature notation), but JMC's chunk parser renders the same array class's - * {@code IMCType.getFullName()} as {@code "byte[]"} (Java source notation), so comparing {@link - * Class#getName()} directly against {@code getFullName()} - as both {@code findMatchForClass} - * overloads above used to - can never match an array-typed leaf class, even when the correct - * event is genuinely present in the recording. Only affects array {@code targetClass} values - * (e.g. {@code byte[].class}); a non-array class's {@code getName()} already matches {@code - * getFullName()} as-is. + * {@code targetClass}'s name in {@code IMCType.getFullName()} format: {@code Class.getName()} + * renders {@code byte[].class} as {@code "[B"}, but JMC renders it as {@code "byte[]"}, so a + * direct comparison never matches an array-typed leaf. Non-array names are unchanged. */ private static String jmcStyleName(Class targetClass) { int dimensions = 0; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java index 9017c3f587..099063d9a1 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java @@ -33,45 +33,27 @@ import static org.junit.jupiter.api.Assertions.fail; /** - * PROF-15341 design doc, Open Question: does JMC's parser actually resolve the {@code - * datadog.ReferenceChain} event's {@code chain} field - declared in jfrMetadata.cpp as {@code - * field("chain", T_CLASS, ..., F_CPOOL | F_ARRAY)}, i.e. an array of scalar - * constant-pool-index {@code T_CLASS} values - the same way it resolves a plain scalar - * F_CPOOL field (e.g. {@code objectClass}, already exercised by {@code AbstractProfilerTest}) - * or a plain F_ARRAY-of-composite-struct field (e.g. {@code jdk.StackTrace#frames})? Neither - * already-exercised shape proves this exact combination. + * Verifies JMC resolves {@code datadog.ReferenceChain}'s {@code chain} field - declared in + * jfrMetadata.cpp as {@code field("chain", T_CLASS, ..., F_CPOOL | F_ARRAY)}, an array of + * constant-pool {@code T_CLASS} indices. Neither scalar F_CPOOL fields (e.g. {@code objectClass}) + * nor F_ARRAY-of-struct fields (e.g. {@code jdk.StackTrace#frames}) cover this combination. * - *

    This test loads the standalone .jfr file produced by the companion gtest ({@code - * ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp}) directly via {@link - * JfrLoaderToolkit} - no live profiler attach, no {@code AbstractProfilerTest} lifecycle - and - * asserts the {@code chain} field resolves to the exact class names that gtest seeded, in the - * exact leaf-to-root order {@code ReferenceChainTracker::buildChainEvent()} (referenceChains.h) - * produces. + *

    Loads the .jfr produced by the companion gtest ({@code referenceChainJfrRoundtrip_ut.cpp}) + * directly, with no live profiler, and asserts the seeded class names in leaf-to-root order. * *

    Why a generic accessor, not {@code Attribute.attr(...)}: JMC's v1 chunk parser - * (internal.parser.v1.ValueReaders.ArrayReader#getContentType()) registers {@code - * UnitLookup.UNKNOWN} as the declared content type for every array field, regardless of - * what its element reader resolves to - this is generic to all array fields, not specific to the - * cpool case. A field registered as UNKNOWN cannot be bound via {@code - * Attribute.attr(id, name, desc, CLASS).getAccessor(type)} (that requires the field's registered - * content type to match). This test instead looks the field up by identifier via {@code - * IType#getAccessorKeys()} and calls {@code IType#getAccessor(IAccessorKey)} directly - the same - * lower-level lookup JMC's own UI uses for fields it has no compile-time-known attribute for. - * This is not a workaround for a missing capability; it is the correct API for an unregistered - * custom field, and it still calls through the very code - * ({@code ArrayReader.read()}/{@code resolve()} delegating per-element to the field's element - * reader, which for {@code chain} is a {@code PoolReader}) that resolves each array element's - * constant-pool index to its class - the actual thing this test exists to prove. + * ({@code ArrayReader#getContentType()}) registers {@code UnitLookup.UNKNOWN} as the content type + * of every array field, so it cannot be bound via a typed {@code Attribute}. Looking it up by + * identifier via {@code IType#getAccessorKeys()} still goes through {@code ArrayReader}'s + * per-element {@code PoolReader} resolution - the thing this test exists to prove. */ public class ReferenceChainJfrParserTest { private static final String EVENT_TYPE = "datadog.ReferenceChain"; /** - * Same path {@code chainRoundtripJfrPath()} in the companion gtest resolves to: the OS temp - * dir (TMPDIR, falling back to /tmp), agreed by both sides rather than by a shared build - * directory - the gtest (ddprof-lib) and this test (ddprof-test) are different Gradle - * modules/tasks with no other filesystem contract between them. + * Must match the companion gtest's {@code chainRoundtripJfrPath()} (TMPDIR, else /tmp): the + * two run in different Gradle modules with no other shared filesystem contract. */ private static Path roundtripJfrPath() { String tmp = System.getenv("TMPDIR"); @@ -83,12 +65,8 @@ private static Path roundtripJfrPath() { public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() throws IOException, CouldNotLoadRecordingException { Path jfrPath = roundtripJfrPath(); - // Assumption, not assertion: the fixture is produced by the companion gtest - // (referenceChainJfrRoundtrip_ut.cpp), which does not run on CI runners - // without Google Test (e.g. the el7 testRelease job) - there the file can - // never exist and this test would fail spuriously. Where the gtest ran in - // the same Gradle invocation (local testDebug, gtest-equipped CI jobs), the - // file exists and the test executes; otherwise it is skipped. + // Assumption, not assertion: the producing gtest does not run on CI runners without Google + // Test (e.g. the el7 testRelease job), where the file can never exist. Assumptions.assumeTrue(Files.exists(jfrPath), "Expected " + jfrPath + " to exist - run " + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " @@ -133,10 +111,9 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() chain.add((IMCType) element); } resolvedChain = chain; - // Retention-edge labels: one per chain hop, aligned with it (the - // gtest seeds kind labels because its JNIEnv is null - the label - // DECODE is skipped; the Java side validates the ARRAY FIELD parses, - // not the decode itself, which referenceChains_ut covers natively). + // Retention-edge labels, one per hop. The gtest has no JNIEnv, so it seeds kind labels + // instead of decoded ones; this checks the array field parses, not the decode + // (covered natively by referenceChains_ut). Object edgesValue = edgesAccessor.getMember(item); assertNotNull(edgesValue, "'edges' field resolved to null"); assertTrue(edgesValue instanceof Object[], diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java index 0d33e8fba2..57107cddaf 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java @@ -14,13 +14,10 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; /** - * PROF-15341 follow-up: {@code ReferenceChainTrackingTest} exercises {@code LivenessTracker}'s - * probabilistic allocation-sampling-driven slope detection and {@code ReferenceChainTracker}'s - * root-seeded BFS discovery together, in one real-JVM run - reliable only when both mechanisms - * happen to line up within the same bounded retry window. This class decouples them via the - * debug-build-only test seams on {@link JavaProfiler} (backed by {@code LivenessTracker}'s - * existing {@code *ForTest} seams and a new {@code ReferenceChainTracker::tagAsRootForTest()}, - * see javaApi.cpp), so each mechanism can be verified end-to-end in isolation: + * {@code ReferenceChainTrackingTest} exercises {@code LivenessTracker}'s probabilistic slope + * detection and {@code ReferenceChainTracker}'s root-seeded BFS discovery together, which is + * reliable only when both line up within the same retry window. This class decouples them via the + * debug-build-only seams on {@link JavaProfiler} (javaApi.cpp), verifying each in isolation: *

      *
    • {@link #shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope()} - asserts a slope signal * would fire from directly-seeded population history, with no allocation sampling involved.
      • @@ -29,20 +26,15 @@ * dependency on {@code selectLeakCandidates()} organically picking the right klass. *
    * - *

    Only runs under the debug native build ({@code testdebug}) - the backing native methods do - * not exist in a release build (see javaApi.cpp's {@code #ifdef DEBUG} guard), mirroring - * {@code JVMAccessTest}'s own {@code "debug".equals(System.getProperty("ddprof_test.config"))} - * pattern for the same reason. + *

    Only runs under the debug native build - the backing native methods do not exist in a + * release build (javaApi.cpp's {@code #ifdef DEBUG} guard). */ public class ReferenceChainTestSeamsTest extends AbstractProfilerTest { - // Arbitrary, test-chosen klass ids - opaque keys for the population table, EXCEPT - // where a representative is involved: setKlassPopulationRepresentativeForTest() now - // aliases the representative's REAL class id onto the seeded synthetic id (the - // leak-tag redesign keys every consumer by real ids), so shouldReconstructChainForDirectlyTaggedRoot() - // must target a class whose real population entry does not already exist - a unique - // nested class, not java.lang.Object - otherwise the real entry's genuine (noisy) - // history outranks the seeded ramp and the target never becomes a candidate. + // Arbitrary klass ids, opaque to the population table except when a representative is set: + // that aliases the representative's real class id onto the synthetic id, so the chain test + // targets a unique nested class - a real entry with existing (noisy) history, e.g. + // java.lang.Object's, would outrank the seeded ramp. private static final int SLOPE_TEST_KLASS_ID = 987001; private static final int CHAIN_TEST_KLASS_ID = 987002; @@ -51,26 +43,18 @@ private static final class ChainTarget {} /** * Durable root for {@link #shouldReconstructChainForDirectlyTaggedRoot()}'s target: the - * noise gate (suppressChainEvent) intentionally drops depth<2 chains whose root is - * transient (stack local / JNI local) - a method-local target is transient BY DESIGN, so - * holding it statically (the canonical leak shape: static field) is required for the - * chain event to be emitted at all. Shared-JVM hygiene: nulled in a finally below so the - * next test in this no-forkEvery JVM does not inherit the reference. + * noise gate (suppressChainEvent) drops depth<2 chains whose root is transient (stack/JNI + * local), so a static holder is required for the chain event to be emitted at all. */ private static Object chainTargetHolder; @Override protected String getProfilerCommand() { - // generations=true: gates LivenessTracker's population tracking (gcGenerationsEnabled()) - - // required for selectLeakCandidates() to return anything at all, real or seeded. - // referencechains=true: constructs ReferenceChainTracker's FrontierTable so - // tagAsReferenceChainRoot0()/runReferenceChainPass0() have a table to insert into. - // framecap=2000000 (not 1024): shouldReconstructChainForDirectlyTaggedRoot()'s first pass - // runs IterateOverReachableObjects, which admits every GC root in the whole JVM - not just - // this test's directly-tagged target - into the same frontier. A small framecap fills with - // that ambient root count before the tagged root ever gets to expandFrontier(), hitting - // SearchAbandonReason::FRONTIER_CAP with no chain event queued. Same value/rationale as - // ReferenceChainTrackingTest's own success-path tests (see that class's header comment). + // generations=true: required for selectLeakCandidates() to return anything, real or seeded. + // referencechains=true: constructs the FrontierTable the seams insert into. + // framecap=2000000: the first pass's IterateOverReachableObjects admits every GC root in the + // JVM, and a small framecap fills with them before the tagged root is expanded, abandoning + // with FRONTIER_CAP and no chain event. return "generations=true,referencechains=true:hops=32:budget=500:ttl=60000:framecap=2000000"; } @@ -96,18 +80,13 @@ public void shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope() { JavaProfiler.resetKlassPopulationForTest0(); JavaProfiler.setGcGenerationsEnabled0(true); - // KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 (livenessTracker.h) is only the floor at which a - // trend becomes eligible at all - selectLeakCandidates() also requires consecutive_positive - // to reach LEAK_TREND_HYSTERESIS_BASE = 5 consecutive qualifying epochs before trusting it - // (the sustained-trend hysteresis gate), so at least 10 + (5 - 1) = 14 strictly increasing - // samples are needed; 20 gives headroom. + // KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 only makes a trend eligible; selectLeakCandidates() + // also needs LEAK_TREND_HYSTERESIS_BASE = 5 consecutive qualifying epochs, so at least + // 10 + (5 - 1) = 14 increasing samples are needed; 20 gives headroom. for (int epoch = 1; epoch <= 20; epoch++) { JavaProfiler.seedKlassPopulationSample0(SLOPE_TEST_KLASS_ID, epoch * 10, epoch); - // Per-(klass, tid) qualification: selectLeakCandidates() now requires a - // qualifying allocating thread on top of the klass-level ramp. This test - // only asserts the klass makes it into the candidate list - it has no - // tracked instances whose tid a tagging scope would need to match - so a - // fixed synthetic tid suffices here. + // selectLeakCandidates() also requires a qualifying allocating thread. No tracked instance + // needs to match here, so a fixed synthetic tid suffices. JavaProfiler.seedTidTrendSample0(SLOPE_TEST_KLASS_ID, 4242, epoch * 3, epoch); } @@ -137,73 +116,42 @@ public void shouldReconstructChainForDirectlyTaggedRoot() { assumeDebugBuild(); JavaProfiler.resetKlassPopulationForTest0(); JavaProfiler.setGcGenerationsEnabled0(true); - // Guards against inheriting a FrontierTable an earlier test in this same, no-forkEvery - // JVM left permanently full/tiny - e.g. ReferenceChainTrackingTest's own - // shouldReportAbandonedSearchOnTinyFrontierCap deliberately drives the shared table to - // framecap=1 and leaves it that way. Same defensive pattern that class's own tests already - // use (see its header comment); without it, tagAsReferenceChainRoot0()'s insert() below can - // fail against a table this test never sized itself. + // Guards against inheriting a full/tiny FrontierTable from an earlier test in this + // no-forkEvery JVM (e.g. ReferenceChainTrackingTest's framecap=1 abandonment test), against + // which tagAsReferenceChainRoot0()'s insert() could fail. JavaProfiler.resetReferenceChainSearchForTest0(); - // The target is referenced ONLY through the static holder - never bound to a - // live local of this frame. A local `target` variable would make the object a - // STACK_LOCAL GC root for the whole test body, and the noise gate (rightly) - // suppresses depth-0 chains rooted in a transient stack slot: the durable - // roots this fixture wants (the static holder, plus the JNI-global-weak-ref - // root LivenessTracker's representative itself creates) then lose the - // durability race only if a stack-local root exists at all. Accessing the - // object only via the static field leaves its frame slots empty between - // calls, so no stack root is ever enumerated. + // Referenced only through the static holder, never a local: a local would make the target a + // stack-local GC root for the whole test body, and the noise gate suppresses chains rooted in + // a transient stack slot. chainTargetHolder = new ChainTarget(); - // Representative BEFORE seeding: with the aliasing seam, setting the representative is - // what registers the synthetic->real id alias, so the seeding below lands re-keyed under - // the target's real class id (seeding first still works - the synthetic entry gets - // re-keyed at representative-set time - but rep-first is the documented load-bearing order). + // Representative before seeding: setting it registers the synthetic->real id alias, so the + // seeds land under the target's real class id. JavaProfiler.setKlassPopulationRepresentativeForTest0(CHAIN_TEST_KLASS_ID, chainTargetHolder); - // See shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope()'s comment above for why 20 - // (not just KLASS_POPULATION_MIN_FILL_FOR_TREND = 10) is needed to clear the hysteresis gate. + // 20 epochs to clear the hysteresis gate - see the slope test above. for (int epoch = 1; epoch <= 20; epoch++) { JavaProfiler.seedKlassPopulationSample0(CHAIN_TEST_KLASS_ID, epoch * 10, epoch); - // Per-(klass, tid) qualification, seeded with THIS thread's real - // profiler tid: the target was allocated here, so if any path in the - // pass/poll cycle below does hinge on leak tagging of the target as a - // tracked instance, the qualifying tid matches it. (The chain's primary - // path is the directly-tagged frontier root + seeded representative, so - // this is belt-and-braces, not a proven requirement.) + // This thread's real tid, since the target was allocated here. Belt-and-braces: the + // chain's primary path is the directly-tagged root plus the seeded representative. JavaProfiler.seedTidTrendSample0( CHAIN_TEST_KLASS_ID, JavaProfiler.getTid(), epoch * 3, epoch); } - // One poll BEFORE the direct tag: the first pollReferenceChainTargets0() - // admits the seeded klass as a watched candidate (canary slot), and the - // seam's discovery recording (tagAsRootForTest's recordDiscoveredInstance) - // only lands when a candidate slot already watches the klass - the - // leak-tag redesign removed the marker-tag path the original pre-poll tag - // used to ride. + // Poll before the direct tag: it admits the seeded klass as a watched candidate, and + // tagAsRootForTest's recordDiscoveredInstance only lands when a candidate slot already + // watches the klass. JavaProfiler.pollReferenceChainTargets0(); - // The target is referenced ONLY through the static holder - never bound to a - // live local of this frame. A local `target` variable would make the object a - // STACK_LOCAL GC root for the whole test body, and the noise gate (rightly) - // suppresses depth-0 chains rooted in a transient stack slot: the durable - // roots this fixture wants (the static holder, plus the JNI-global-weak-ref - // root LivenessTracker's representative itself creates) then lose the - // durability race only if a stack-local root exists at all. Accessing the - // object only via the static field leaves its frame slots empty between - // calls, so no stack root is ever enumerated. long tag = JavaProfiler.tagAsReferenceChainRoot0(chainTargetHolder); assertTrue(tag > 0, "Expected tagAsReferenceChainRoot0 to assign a valid frontier tag"); - // Drive pass+poll cycles until the chain event lands. One pass is NOT enough: - // admitStaticFieldRoots() sweeps loaded classes in chunks (a few hundred per pass, - // bounded by the budget) and the first pass truncates before reaching this test's - // class - the durable (static-field) root discovery that upgrades the target's - // root_kind only happens on the pass whose chunk covers ChainTarget's holder class. - // The bound (40) gives the full sweep lap (~10 passes for ~4000 loaded classes) - // several round trips of headroom; the BFS thread is excluded from racing these - // calls by the engine lock (runPassSerialized/pollWatchedTargetsSerialized). + // One pass is not enough: admitStaticFieldRoots() sweeps loaded classes in budget-bounded + // chunks, and the target's durable root_kind is only discovered on the pass whose chunk + // covers ChainTarget's holder class. 40 passes give several full sweep laps of headroom + // (~10 passes for ~4000 classes). The engine lock (runPassSerialized/ + // pollWatchedTargetsSerialized) keeps the BFS thread from racing these calls. int eventCount = 0; for (int pass = 0; pass < 40 && eventCount == 0; pass++) { boolean sawPassRun = JavaProfiler.runReferenceChainPass0(); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java index 9506c2b9d6..0d057ce48c 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java @@ -33,118 +33,57 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * PROF-15341 (+ lifecycle-wiring follow-up, + the Remaining Work Plan's target-selection bridging, - * pause-time pacing, and reporting work): end-to-end - * coverage for {@code ReferenceChainTracker} (ddprof-lib/src/main/cpp/referenceChains.h/.cpp). + * End-to-end coverage for {@code ReferenceChainTracker} (referenceChains.h/.cpp) against a real + * JVM, real GCs and a real JVMTI heap walk. {@code Profiler::dump()} emits + * {@code datadog.ReferenceChainAbandoned} once the search has ended in + * {@code SearchState::ABANDONED}; {@code pollWatchedTargets()} emits {@code datadog.ReferenceChain} + * for ranked {@code selectLeakCandidates()} klasses whose representative a {@code runPass()} walk + * has already tagged ({@code getTag() > 0} - a read, never a {@code SetTag} seed). * - *

    Scope note: {@code ReferenceChainTracker::start()} is now called from - * {@code Profiler::start()} (profiler.cpp), gated on {@code args._reference_chains}, followed by - * {@code ReferenceChainTracker::startThread()} which spawns its BFS thread; {@code Profiler::stop()} - * calls the matching {@code stopThread()}/{@code stop()} pair. A {@code datadog.ReferenceChainAbandoned} - * event now reaches a live {@code Recording}: {@code Profiler::dump()} calls - * {@code buildAbandonedEvent()} and writes its output via - * {@code Profiler::writeReferenceChainAbandoned()}/{@code FlightRecorder::recordReferenceChainAbandoned()} - * whenever the search has ended in {@code SearchState::ABANDONED} - mirroring the - * {@code LivenessTracker::flush()} call site there. {@link #shouldReportAbandonedSearchOnTinyFrontierCap()} - * exercises that path end-to-end below. - * - *

    {@code buildChainEvent()}'s target-selection feed (Remaining Work Plan's target-selection - * bridging step): - * {@code ReferenceChainTracker::pollWatchedTargets()} (referenceChains.cpp), called from - * {@code threadLoop()} once per scheduling cycle after {@code runPass()}, closes the gap this class's - * header comment used to describe as unclosed. It polls - * {@code LivenessTracker::selectLeakCandidates()} (positive population-slope ranking over a rolling - * per-klass survivor-count window, gated on {@code _gc_generations} - design doc's Open Question 3) - * and, for each ranked klass whose representative instance an ordinary {@code runPass()} walk has - * *already* tagged ({@code getTag() > 0} - a read, never a {@code SetTag} seed), reconstructs and - * emits its chain via {@code Profiler::writeReferenceChain()}. {@link #shouldReconstructReferrerChainToGcRoot()} - * below exercises this end to end against a real JVM, real GCs, and a real JVMTI heap walk - not a - * synthetic frontier fixture (see {@code referenceChainJfrRoundtrip_ut.cpp}/{@code ReferenceChainJfrParserTest} - * for that already-covered, lower-level reconstruction-correctness proof). - * - *

    Why {@code @TestMethodOrder}/{@code @Order(1)}: {@code ReferenceChainTracker} is a - * process-wide singleton whose single, singleton-owned search (design doc's Open Question 3 - * "Shipped" note) never restarts once it leaves {@code SearchState::RUNNING} - - * {@code shouldRunPass()} (referenceChains.cpp) returns {@code false} forever after that point, and - * the transition itself releases every tag the search ever assigned - * ({@code releaseSearchTags()}). Its {@code FrontierTable} is sized once, the first time any test - * in this JVM calls {@code ReferenceChainTracker::start()}, and never resized on later - * start()/stop() cycles (referenceChains.cpp's {@code if (_frontier == nullptr)} guard) - so - * whichever test runs first also fixes that capacity for every test that runs after it in the same - * JVM (no {@code forkEvery} configured, see {@code ProfilerTestPlugin.kt}, so this whole class runs - * in one). {@link #shouldReportAbandonedSearchOnTinyFrontierCap()} below deliberately drives that - * shared search to {@code SearchState::ABANDONED}, via an artificially tiny frontier cap when it - * gets to build the table itself, or its own {@code ttl} fallback otherwise (see that method's own - * comment). Both {@link #shouldReconstructReferrerChainToGcRoot()} and - * {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()} need the search still - * {@code RUNNING} to find their own candidates' tags non-zero, so they are pinned to run first, - * via {@code @Order(1)}/{@code @Order(2)} respectively - without that ordering all three tests - * would race for which ones get to observe a still-{@code RUNNING} search, and neither - * success-path test has a fallback for losing that race the way the abandonment test does. The - * relative order between the two success-path tests does not itself matter - both target - * different klasses ({@link ChainLink} vs {@link CachedPayload}) within the same shared search, - * and neither exhausts it - only their both running before the abandonment test does. + *

    Why {@code @TestMethodOrder}: {@code ReferenceChainTracker} is a process-wide + * singleton. Leaving {@code SearchState::RUNNING} releases every tag the search assigned + * ({@code releaseSearchTags()}), and its {@code FrontierTable} is sized only by the first + * {@code start()} in this JVM ({@code if (_frontier == nullptr)} guard; no {@code forkEvery}, see + * {@code ProfilerTestPlugin.kt}). {@link #shouldReportAbandonedSearchOnTinyFrontierCap()} drives + * the search to {@code ABANDONED}, while both success-path tests need it {@code RUNNING} to see + * non-zero tags and have no fallback for losing that race, so they run first + * ({@code @Order(1)}/{@code @Order(2)}). Their relative order does not matter: they target + * different klasses ({@link ChainLink} vs {@link CachedPayload}) and neither exhausts the search. */ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @Tag("slow") public class ReferenceChainTrackingTest extends AbstractProfilerTest { - // Arbitrary, test-chosen klass ids for the debug-only population-seeding seams below (see - // ReferenceChainTestSeamsTest's own comment: LivenessTracker's population table treats these as - // opaque keys, so they need not resolve to any real class). Distinct per test/from - // ReferenceChainTestSeamsTest's own ids purely as cheap insurance against collision in a shared, - // no-forkEvery test JVM. + // Arbitrary klass ids for the debug-only population-seeding seams. Distinct from each other and + // from ReferenceChainTestSeamsTest's ids to avoid collisions in the shared, no-forkEvery JVM. private static final int CHAIN_LINK_TEST_KLASS_ID = 987201; private static final int CACHED_PAYLOAD_TEST_KLASS_ID = 987202; - // Durable (static-field) holder for the gc-root fixture below. The - // discovered-chain gate suppresses chains shallower than the first real - // holder hop that are rooted at a TRANSIENT root (stack/JNI local) - a - // frame-held "leak" is by definition not a leak - and - // shouldReconstructReferrerChainToGcRoot()'s ChainLink instances sit at - // depth 1 of a locally-held list, exactly the transient-rooted shape the - // gate exists to suppress. A static field gives them the real - // unmaintained-singleton-collection retention shape instead. (The cache - // fixture below needs no such change: its CachedPayload chains are three - // hops deep through HashMap's own internals, which the gate correctly - // lets through regardless of root kind.) Cleared in the test's finally: - // this is a shared, no-forkEvery test JVM, and a leftover holder would - // keep its entire population reachable for every later test here. + // Static, not local: the discovered-chain gate suppresses shallow chains rooted at a transient + // root (stack/JNI local), and ChainLinks at depth 1 of a locally-held list are exactly that + // shape. (CachedPayload chains go three hops through HashMap internals, which the gate lets + // through regardless of root kind.) Cleared in the test's finally so it does not keep its + // population reachable for later tests in this shared JVM. private static List gcRootHolder; - // The current round's allocation batch, handed to the persistent allocator - // thread below. Volatile: written here, read by the allocator thread every - // round - the request queue's happens-before (offer after set, take before - // read) already covers it, but the visibility of a plain field handed - // across threads is kept explicit rather than relying on queue semantics - // alone. + // Volatile: handed to the allocator thread every round. The request queue already provides + // happens-before; this keeps the cross-thread visibility explicit. private static volatile List currentBatch; - // LivenessTracker's own hysteresis gate (livenessTracker.h's - // KLASS_POPULATION_MIN_FILL_FOR_TREND=10 / LEAK_TREND_HYSTERESIS_BASE=5; - // livenessTracker.cpp's recordKlassPopulationSampleLocked()/hasQualifyingGrowth()) only starts - // incrementing KlassPopulationEntry::consecutive_positive once ring_fill has reached 10 - every - // call before that resets it straight back to 0 (ringThirdsStats()'s own fill < min_fill guard). - // A single batch of exactly 10 seedKlassPopulationSample0() calls therefore caps - // consecutive_positive at 1 (only the 10th call ever sees ring_fill==10), far short of the >=5 - // (or >=3 if LivenessTracker::heapFloorRising() independently corroborates) that - // selectLeakCandidates() (livenessTracker.cpp) requires before it reports a candidate at all - - // which is what ReferenceChainTracker::hasLeakSignal() consults, and what shouldRunPass() - // (referenceChains.cpp) gates even this search's very first pass on whenever generations=true - // (see this class's own header comment on why runPass() never otherwise fires here without one). - // 10 (KLASS_POPULATION_MIN_FILL_FOR_TREND) + 5 (LEAK_TREND_HYSTERESIS_BASE) - 1 = 14 calls are - // the minimum for the 5 qualifying calls at ring_fill>=10 (epochs 10-14) to actually land; +1 - // for margin against off-by-one uncertainty in that derivation. + // consecutive_positive only starts incrementing once ring_fill reaches + // KLASS_POPULATION_MIN_FILL_FOR_TREND=10 (ringThirdsStats()'s fill < min_fill guard resets it to + // 0 before that), and selectLeakCandidates() needs it at LEAK_TREND_HYSTERESIS_BASE=5 (or 3 when + // heapFloorRising() corroborates). hasLeakSignal() - which shouldRunPass() gates even the first + // pass on under generations=true - consults that. 10 + 5 - 1 = 14 seeded epochs is the minimum; + // +1 for margin. private static final int SEED_EPOCHS_FOR_HYSTERESIS = 15; - // Dedicated id for shouldReportAbandonedSearchOnTinyFrontierCap()'s own seeding (see that - // method's own comment) - distinct from CHAIN_LINK_TEST_KLASS_ID/CACHED_PAYLOAD_TEST_KLASS_ID - // for the same cheap-insurance-against-collision reason those two are distinct from each other. + // Dedicated id for shouldReportAbandonedSearchOnTinyFrontierCap()'s seeding, for the same + // collision-avoidance reason as the ids above. private static final int ABANDON_TEST_KLASS_ID = 987203; - // T_REFERENCE_CHAIN in ddprof-lib/src/main/cpp/jfrMetadata.h - the jdk.ActiveSetting "id" - // the profiler writes for the datadog.ReferenceChain setting (flightRecorder.cpp's - // writeBoolSetting(buf, T_REFERENCE_CHAIN, "enabled", ...)). Must stay in sync with that enum. + // T_REFERENCE_CHAIN in jfrMetadata.h - the jdk.ActiveSetting "id" written by flightRecorder.cpp's + // writeBoolSetting(buf, T_REFERENCE_CHAIN, "enabled", ...). Must stay in sync with that enum. private static final long REFERENCE_CHAIN_EVENT_ID = 129; @Override @@ -154,121 +93,59 @@ protected String getProfilerCommand() { : ""; if ("shouldReconstructReferrerChainToGcRoot".equals(testName) || "shouldReconstructReferrerChainThroughUnboundedCacheLeak".equals(testName)) { - // memory=...:l + generations=true: LivenessTracker::gcGenerationsEnabled() (Remaining - // Work Plan's population tracking / target-selection bridging, livenessTracker.h) must be true for pollWatchedTargets() - // (referenceChains.cpp) to do anything at all - referencechains=... alone stays - // whole-graph-only (design doc's Open Question 3 "still undecided" fallback). A small - // sampling interval maximizes the chance the many ChainLink instances this test - // allocates below actually get liveness-tracked (LivenessTracker::track(), invoked from - // the allocation-sampling path whenever _gc_generations || _record_liveness is set, - // objectSampler.cpp). A large framecap avoids an early - // SearchAbandonReason::FRONTIER_CAP abandonment against this JVM's real, - // not-controlled-by-this-test root-reachable graph size, and a long ttl avoids a - // TTL abandonment mid-test - either would call releaseSearchTags() and permanently - // zero every tag this test depends on (see this class's header comment on why this - // test is pinned to @Order(1)). + // generations=true: pollWatchedTargets() does nothing unless + // LivenessTracker::gcGenerationsEnabled() is true; referencechains=... alone stays + // whole-graph-only. A large framecap avoids FRONTIER_CAP abandonment against this JVM's + // real root-reachable graph, and a long ttl avoids TTL abandonment mid-test - either would + // call releaseSearchTags() and zero every tag this test depends on. // - // pausetarget=5000: the pause-time pacing controller's updatePacing() (referenceChains.cpp) measures this JVM's real - // FollowReferences/GetObjectsWithTags call latency at ~150ms early on, climbing past a full - // second as this test's own allocation keeps handing the search more to discover (each pass - // re-scans the whole already-discovered/tagged set, so cost grows with cumulative progress, - // not with this test's own allocation *rate*) - i.e. that latency is dominated by per-call - // overhead, not by the requested edge budget. Against the default pausetarget=5ms ceiling - // (DEFAULT_REFERENCE_CHAINS_PAUSE_TARGET_MS, arguments.h) every observed pass is "over - // ceiling even at the floor", so the controller settles at MIN_EFFECTIVE_BUDGET=50 edges/pass - // and MAX_EFFECTIVE_CADENCE_NS=4s between passes - a real but glacial discovery rate this - // test's own wall-clock budget cannot outwait, and one a merely-generous ceiling (e.g. 500ms) - // only postpones: once cumulative progress pushes the real per-pass cost back past *that* - // ceiling too, the same throttling recurs. A ceiling comfortably above the highest per-call - // cost this test's own scale of population growth is expected to reach instead lets the - // controller keep _effective_budget at this test's own configured budget=4000 ceiling for - // the method's entire run, the same convergence behavior referenceChains_ut.cpp's own pacing - // tests already exercise synthetically. + // pausetarget=5000: measured FollowReferences/GetObjectsWithTags latency starts ~150ms and + // climbs past 1s as cumulative progress grows (each pass re-scans the discovered set). + // Against the default 5ms ceiling (DEFAULT_REFERENCE_CHAINS_PAUSE_TARGET_MS) updatePacing() + // settles at MIN_EFFECTIVE_BUDGET=50 edges/pass and MAX_EFFECTIVE_CADENCE_NS=4s, too slow + // for this test's wall-clock budget; a merely generous ceiling only postpones that. A + // ceiling above the expected peak per-pass cost keeps _effective_budget at budget=4000. // - // painbudget=100: shouldReconstructReferrerChainThroughUnboundedCacheLeak shares this - // singleton search with shouldReconstructReferrerChainToGcRoot (@Order(1), runs first) and - // needs a restart (canAffordNewSearch(), referenceChains.cpp) once that first search - // reaches a terminal state. PainBudget (painBudget.h) gates that restart on - // pain_spent_ms / refill_rate milliseconds of cooldown; the default - // DEFAULT_REFERENCE_CHAINS_PAIN_BUDGET_PERCENT=1 (1% refill) can make that cooldown run to - // tens of seconds to minutes for a search this large - far past this test's own bounded - // retry loop (16 rounds plus a short grace period). painbudget=100 keeps the restart gate - // affordable immediately, which is fine here since nothing else in this JVM is competing - // for the pain budget. + // painbudget=100: the @Order(2) test needs a search restart (canAffordNewSearch()) after + // the @Order(1) search terminates. PainBudget (painBudget.h) gates that on + // pain_spent_ms / refill_rate of cooldown, which at the default 1% refill can run to + // minutes - past this test's bounded retry loop. Nothing else here competes for the budget. // - // firstpassbudget=200000: root/stack-ref enumeration (runPassManualWalk()'s - // IterateOverReachableObjects call) shares budget=4000 with expandFrontier()'s steady-state - // expansion unless overridden - in this shared, forked test JVM's real (not controlled by - // this test) root-reachable graph, that leaves this method's own CachedPayload-holding - // cache HashMap (a direct stack-local root on this test's own thread) at the mercy of - // whatever order JVMTI's enumeration happens to visit roots in: budget=4000 was observed - // exhausting itself on other roots' subtrees every single attempt, never once admitting a - // single edge from this thread's own stack. Overriding just root enumeration's own budget - // (not budget=4000, which the pacing controller has already been tuned around - raising it - // instead throttles every expansion pass down to - // MIN_EFFECTIVE_BUDGET/MAX_EFFECTIVE_CADENCE_NS, see updatePacing()) gives every root - // enumeration attempt (the first pass, and any later one ROOT_ENUM_MIN_INTERVAL_NS gates - // back in - see that constant's own comment) enough headroom to reach this thread's stack - // without slowing down the steady-state expansion passes that follow. - // memory=64:l:1.0 - the explicit 100% live-sample ratio is load-bearing: - // without a ratio segment the default is 10% (arguments.h's - // _live_samples_ratio) and LivenessTracker::track() drops 90% of - // tracked instances probabilistically, thinning the pool the - // representative-minting and candidate trend rings draw from - - // observed live as the intermittent "representative died/evicted" - // and zero-tag runs (see LeakTagCorrelationReferenceChainTest's own - // comment for the full lottery analysis). Total tracking volume is - // still capped by the 256KiB sampling-interval floor, not the ratio, - // so 100% keeps the tracking table tiny. + // firstpassbudget=200000: root enumeration (runPassManualWalk()'s + // IterateOverReachableObjects) otherwise shares budget=4000, which was observed exhausting + // on other roots' subtrees before ever admitting an edge from this thread's stack (where + // the cache test's HashMap lives). Raising only the root-enumeration budget leaves the + // steady-state expansion pacing (updatePacing()) untouched. + // + // memory=64:l:1.0 - the explicit 100% live-sample ratio is load-bearing: the default 10% + // (_live_samples_ratio, arguments.h) makes LivenessTracker::track() drop 90% of tracked + // instances, thinning the representative and trend pools (observed as intermittent + // "representative died/evicted" and zero-tag runs). Tracking volume is still capped by the + // 256KiB sampling-interval floor, so the table stays small. return "memory=64:l:1.0,generations=true," + "referencechains=true:hops=64:budget=4000:ttl=120000:framecap=2000000:pausetarget=5000:painbudget=100:firstpassbudget=200000"; } - // shouldReportAbandonedSearchOnTinyFrontierCap needs the frontier table (not the - // per-pass budget) to be what runs out first: heapReferenceCallback() (referenceChains.cpp) - // checks the budget before ever calling FrontierTable::insert(), so a budget as small as - // the frontier cap itself (e.g. budget=1:framecap=1) only ever exhausts the budget after - // admitting exactly one object - insert() never gets a chance to fail, and the search - // completes normally instead of abandoning. A budget comfortably larger than framecap=1 - // lets a *second* root-referenced object reach insert() and hit the actually-full table, - // which is what runPass() treats as grounds to abandon (Termination section priority 1). + // shouldReportAbandonedSearchOnTinyFrontierCap needs the frontier table, not the per-pass + // budget, to run out first: heapReferenceCallback() checks the budget before + // FrontierTable::insert(), so budget<=framecap completes normally instead of abandoning. A + // larger budget lets a second root-referenced object hit the full table, which runPass() + // abandons on. // - // framecap=1 only actually controls the table's capacity the *first* time - // ReferenceChainTracker::start() ever constructs its FrontierTable in this JVM - // (referenceChains.cpp's `if (_frontier == nullptr)` guard - the table, like - // LivenessTracker's own, survives every later start()/stop() cycle at whatever size it was - // first built with). This method's own @Order(1) pinning on - // shouldReconstructReferrerChainToGcRoot means *that* test's own, much larger framecap=2000000 - // wins that race instead - so this method also supplies a small ttl as a second, independent - // abandonment trigger. runPass()'s Termination-section priority still checks frontier-cap - // first, so a fresh, genuinely-tiny table (this method running first, e.g. in isolation via - // `-Ptests=ReferenceChainTrackingTest.shouldReportAbandonedSearchOnTinyFrontierCap`) still - // abandons via SearchAbandonReason::FRONTIER_CAP as originally designed; inheriting an - // already-oversized table instead falls through to the ttl check, which fires almost - // immediately regardless of table size because _search_start_ns (referenceChains.h) is set - // once, the first time the shared search ever started - by the time this method's own - // start() call re-parses ttl, that clock already reads however long - // shouldReconstructReferrerChainToGcRoot's own run took. Either path produces a - // datadog.ReferenceChainAbandoned event, which is all this method actually asserts on. + // framecap=1 only sizes the table if this is the first start() in the JVM + // (`if (_frontier == nullptr)` guard); otherwise @Order(1)'s framecap=2000000 wins. ttl=100 is + // the fallback trigger: _search_start_ns is set when the shared search first started, so on an + // inherited oversized table the ttl check fires almost immediately. Either path produces + // datadog.ReferenceChainAbandoned, which is all this method asserts on. // - // painbudget=100, for the same reason shouldReconstructReferrerChainThroughUnboundedCacheLeak() - // above needs it (see that method's own comment on painbudget=100): this method's own - // resetReferenceChainSearchForTest0() call spends whatever _search_pain_ms - // shouldReconstructReferrerChainThroughUnboundedCacheLeak()'s own long-running search left - // behind (found the hard way - that search runs 20+ passes admitting tens of thousands of - // edges each, and nothing zeroes _search_pain_ms between its own end and this method's start() - // reconstructing a fresh, zero-balance PainBudget) into *this* method's own PainBudget - // (referenceChains.cpp's restartSearch()/resetSearchStateForTest(), both call - // _pain_budget.spend(_search_pain_ms) unconditionally) - at the default 1% refill rate, - // draining that debt back to canStartNow()==true would take on the order of minutes, far past - // this method's own retry budget below. painbudget=100 keeps that gate affordable immediately, - // the same way it does for the sibling method. + // painbudget=100: restartSearch()/resetSearchStateForTest() both call + // _pain_budget.spend(_search_pain_ms) unconditionally, so the reset below inherits the + // previous long-running search's pain debt; at the default 1% refill, draining it back to + // canStartNow()==true takes minutes, past this method's retry budget. // - // memory=64:l:1.0 is load-bearing: Profiler::start() (profiler.cpp) only starts the - // ReferenceChainTracker when allocation sampling actually activated (activated & EM_ALLOC). - // A bare referencechains=... command falls back to cpu-only profiling, so the search never - // runs and dump() could never emit datadog.ReferenceChainAbandoned. No generations=true: - // LivenessTracker::initialize() then clears _gc_generations, so hasLeakSignal() returns true - // without needing any population trend. + // memory=64:l:1.0 is load-bearing: Profiler::start() only starts the ReferenceChainTracker + // when allocation sampling activated (activated & EM_ALLOC); a bare referencechains=... falls + // back to cpu-only and never runs the search. No generations=true: LivenessTracker::initialize() + // then clears _gc_generations, so hasLeakSignal() returns true without any population trend. if ("shouldReportAbandonedSearchOnTinyFrontierCap".equals(testName)) { return "memory=64:l:1.0,referencechains=true:hops=32:budget=500:framecap=1:ttl=100:painbudget=100"; } @@ -289,7 +166,7 @@ protected boolean isPlatformSupported() { /** * Verifies the {@code referencechains=...} flag round-trips through * {@code Arguments} parsing (arguments.cpp's {@code CASE("referencechains")}) into the - * {@code datadog.ReferenceChain} JFR setting (flightRecorder.cpp:1143's + * {@code datadog.ReferenceChain} JFR setting (flightRecorder.cpp's * {@code writeBoolSetting(buf, T_REFERENCE_CHAIN, "enabled", args._reference_chains)}). */ @RetryingTest(5) @@ -309,85 +186,44 @@ public void shouldExposeReferenceChainsSettingWhenEnabled() { } /** - * This test's stated success-path scenario, now exercising the Remaining Work Plan's - * target-selection bridging feed instead of staying disabled: allocates a growing population of - * {@link ChainLink} instances, forcing GCs between allocation rounds so - * {@code LivenessTracker::cleanup_table()}'s epoch-advance pass (livenessTracker.cpp) observes a - * rising per-klass survivor count each time, until {@code selectLeakCandidates()} trusts the - * resulting trend (needs {@code KLASS_POPULATION_MIN_FILL_FOR_TREND = 10} ring-buffer samples). - * Then waits for {@code ReferenceChainTracker::pollWatchedTargets()} to notice that ranked - * candidate has already been tagged by an ordinary {@code runPass()} walk and reconstruct + - * emit its chain, and asserts on the resulting {@code datadog.ReferenceChain} event. + * Allocates a growing, statically-held population of {@link ChainLink} instances with a GC and + * dump between rounds until {@code selectLeakCandidates()} trusts the trend, then waits for + * {@code pollWatchedTargets()} to emit the candidate's chain once a {@code runPass()} walk has + * tagged it. * - *

    Each population-growth round pairs with an explicit {@link #dump(Path)}: per - * livenessTracker.cpp, {@code cleanup_table()}'s per-klass population accounting only runs - * (unforced) from {@code flush_table()}, which only runs from {@code LivenessTracker::flush()}, - * which is called *only* from {@code Profiler::dump()} - there is no timer-driven flush. - * {@code System.gc()} bumps {@code LivenessTracker::_gc_epoch} synchronously inside the - * {@code GarbageCollectionFinish} callback ({@code onGC()}), so by the time {@code System.gc()} - * returns to Java the epoch bump is already visible - no extra sleep is needed between the - * {@code gc()} and the {@code dump()} that observes it. + *

    Each round needs an explicit {@link #dump(Path)}: {@code cleanup_table()}'s per-klass + * population accounting only runs (unforced) via {@code flush_table()} from + * {@code LivenessTracker::flush()}, which is called only from {@code Profiler::dump()} - there + * is no timer-driven flush. {@code System.gc()} bumps {@code _gc_epoch} synchronously in the + * {@code GarbageCollectionFinish} callback, so no sleep is needed between the GC and the dump. */ @Test @Order(1) public void shouldReconstructReferrerChainToGcRoot() throws Exception { - // Seed gcRootHolder with one live element *before* resetting the search below: ArrayList's - // backing array starts out as the shared empty-array sentinel, and only gets replaced with a - // real array on its first grow() (addAll()). ReferenceChainTracker::expandFrontier() - // (referenceChains.cpp) freezes whatever children it observes for a node the moment that - // node is expanded - it never re-examines an already-EXPANDED node for a later field mutation, - // regardless of how many times root/stack-ref enumeration itself reruns over the search's - // lifetime (runPassManualWalk()'s own comment). If gcRootHolder's node gets expanded while - // elementData still pointed at the empty sentinel (nothing stops some *unrelated* GC in this - // shared, no-forkEvery JVM from waking the freshly (re)started BFS thread in the gap between - // resetting the search below and round 1's own addAll()), the frozen children set would be - // permanently empty, and no ChainLink added in any of the 16 rounds that follow would ever - // become reachable - matching a real CI failure where pollWatchedTargets() reported tag=0 for - // this klass on every single poll across the whole test. Seeding one element first means the - // backing array is never empty at any point after the search (re)starts: every later - // addAll()/grow() copies all prior elements (including this one) into the new array, so - // whichever array the walk happens to snapshot when it expands the node, index 0 is always - // present in it. + // Seed one element *before* resetting the search: ArrayList's backing array starts as the + // shared empty sentinel, and expandFrontier() freezes a node's children when it is expanded + // and never re-examines it. An unrelated GC can wake the restarted BFS thread before round 1's + // addAll(); expanding the holder while still empty would leave every later ChainLink + // unreachable (seen in CI as tag=0 on every poll). Every later grow() copies index 0, so + // whichever array the walk snapshots is non-empty. gcRootHolder = new ArrayList<>(); gcRootHolder.add(new ChainLink("gc-root-seed")); if ("debug".equals(System.getProperty("ddprof_test.config"))) { - // Being pinned to run first *within this class* (this class's own header comment, "Why - // @TestMethodOrder/@Order(1)") does not guarantee this is the first reference-chain test - // to ever call into the shared, process-wide singleton ReferenceChainTracker in this whole - // test JVM - no forkEvery is configured (same header comment), so another test class can - // run first and leave the singleton search already non-RUNNING/already-tagged, in which - // case runPass() never takes its real root-seeded-walk branch again. Force a genuine fresh - // search so this test's own ChainLink population is guaranteed reachable by a real walk, - // regardless of what ran earlier in this JVM. Debug-only: this native seam does not exist - // in a release build. - // - // resetKlassPopulationForTest0() first, for the same "no forkEvery" reason: - // LivenessTracker's _klass_population table (livenessTracker.cpp) also survives stop()/ - // start() cycles, so an unrelated earlier test class in this same shared JVM (e.g. another - // generations=true scenario) could leave a klass already past hasLeakSignal()'s - // consecutive_positive hysteresis gate (selectLeakCandidates(), livenessTracker.cpp) - // sitting in that table - which would let this test's first pass start for a reason that - // has nothing to do with the seeding this method does further down (see - // SEED_EPOCHS_FOR_HYSTERESIS's own comment), masking whether that seeding is actually - // sufficient on its own. + // @Order(1) only orders this class: another test class in this no-forkEvery JVM may have + // left the singleton search non-RUNNING, after which runPass() never takes its root-seeded + // walk again. The population reset is needed for the same reason: _klass_population + // survives stop()/start(), and a leftover klass already past the hysteresis gate would let + // the first pass start regardless of this method's own seeding, masking whether it is + // sufficient. Debug-only: these native seams do not exist in a release build. JavaProfiler.resetKlassPopulationForTest0(); JavaProfiler.resetReferenceChainSearchForTest0(); } - // One persistent allocator thread, reused by every round - NOT a fresh - // spawned thread per round. Same JVMTI property the per-round spawn - // relied on (SampledObjectAlloc never fires for this JUnit worker - // thread's own allocations, but reliably fires for threads created - // after Profiler::start() already ran), plus the decisive per-tid - // consequence: LivenessTracker's per-(klass, tid) qualification trend - // accumulates REAL samples for this one thread's tid every round - // (foldKlassCountsLocked() folds each tracked instance's tid), so - // qualification is organic and cannot age out mid-test. The previous - // per-round spawns gave every round a DIFFERENT short-lived tid, so - // the seeded (klass, worker-tid) qualification ramp was the only - // qualifying signal - and once its ring slots aged out (observed - // live: slope decayed to ~0 at ring_fill=22, consecutive_positive - // reset, the candidate never returned again) the candidate dropped - // out of selectLeakCandidates() with the whole crawl still unfinished. + // One persistent allocator thread, reused by every round: SampledObjectAlloc does not fire + // for this JUnit worker thread's own allocations, but does for threads created after + // Profiler::start(). Keeping a single tid lets the per-(klass, tid) qualification trend + // accumulate real samples every round (foldKlassCountsLocked()), so qualification cannot + // age out mid-test the way a fresh short-lived tid per round did (slope decayed to ~0 at + // ring_fill=22 and the candidate dropped out with the crawl unfinished). LinkedBlockingQueue allocRequests = new LinkedBlockingQueue<>(); Semaphore roundDone = new Semaphore(0); AtomicLong allocatorTid = new AtomicLong(-1); @@ -415,42 +251,20 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { Path scratchDumpPath = Paths.get("referencechains-population-scratch.jfr"); try { - // Up to 16 rounds: selectLeakCandidates()'s KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 - // (livenessTracker.h) needs 10 *epochs that actually observe a surviving ChainLink sample*, - // not just 10 dump() calls - allocation sampling is probabilistic (see below), so some - // early, smaller rounds may not land a single ChainLink sample. Growing the per-round count - // compensates, and this loop keeps going (checking for the event every round) rather than - // committing to a fixed round count up front, so it self-adjusts to whatever this JVM's - // actual sampling behavior turns out to be. Staying under KLASS_POPULATION_RING_SIZE = 30 - // keeps every round's sample within the trend computeKlassPopulationSlope() reads back out. - // Capped at 16 rather than a larger margin above the 10-round minimum: this loop's own - // worst case (every round retained, no early match) runs inside the same forked test JVM - // every other ddprof-test class shares (ProfilerTestPlugin.kt's shared -Xmx512m default, - // no forkEvery) - a prior CI run hit "Java heap space" in that shared fork with this loop - // capped at 25, so the cap trades some of the original margin above the 10-round minimum - // for staying inside that shared heap. + // KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 needs 10 epochs that actually observe a surviving + // ChainLink sample, and sampling is probabilistic, so the loop grows per-round counts and + // checks every round instead of committing to a fixed count. Staying under + // KLASS_POPULATION_RING_SIZE = 30 keeps every round within the slope window. Capped at 16 + // because the shared fork (-Xmx512m, no forkEvery) hit "Java heap space" at 25. // - // Per-round size growth itself is clamped to round 10 (Math.min(round, 10) below): rounds - // past 10 exist only to give pollWatchedTargets() more retries against the lock-contention - // race described below, not to keep building the population trend (already eligible by - // round 10) - letting round*600 keep scaling unclamped through round 16 made rounds 11-16 - // each add strictly more retained garbage than the last for no trend benefit, which is what - // drove the "Java heap space" failure this comment's own history refers to. + // Growth is clamped at round 10: later rounds only add retries against the lock-contention + // race below (the trend is already eligible), and unclamped growth was what exhausted the + // heap. // - // Instance count is sized against ObjectSampler::check() (objectSampler.cpp), not this - // test's own "memory=64" request: "do not allow shorter interval than 256KiB" means the - // *actual* allocation-sampling interval is 262144 bytes regardless of the small value - // requested above (that request only affects LivenessTracker's own table-capacity formula, - // livenessTracker.cpp's initialize_table()). ChainLink's own padding fields (see that - // class's comment) get enough total megabytes sampled from far fewer instances than plain, - // unpadded ~40-byte instances would need - keeping this test's own contribution to the real - // JVM's root-reachable graph (which ReferenceChainTracker's BFS walk must also traverse, - // Triggering section) from ballooning to the point the walk can't practically catch up - // within this test's own wall-clock budget. ChainLink competes on equal footing against - // other incidental klasses (e.g. "[B"/byte[]) that this same mechanism may also legitimately - // flag as leak candidates - this test's own assertions below look for ChainLink specifically - // among however many datadog.ReferenceChain events actually appear, rather than assuming it - // is the only one. + // Sized against ObjectSampler::check()'s 256KiB minimum interval, not "memory=64" (which + // only affects LivenessTracker's table capacity). ChainLink's padding keeps the instance + // count - and so the graph the BFS walk must traverse - small. Other klasses (e.g. byte[]) + // may also be flagged, so matching is by ChainLink specifically. ReferenceChainAssertions.JfrChainMatch match = null; boolean seededTestKlassTrend = false; int totalRounds = 16; @@ -469,63 +283,28 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); if (match == null && "debug".equals(System.getProperty("ddprof_test.config"))) { - // Debug-only native seams - a no-op in release builds, so this leaves release-build - // coverage of the organic path below completely unchanged. + // Under generations=true, shouldRunPass() gates even the first pass on hasLeakSignal(), + // which this early falls through to selectLeakCandidates()'s hysteresis gate for some + // klass. Seeding SEED_EPOCHS_FOR_HYSTERESIS epochs clears it, authorizing the BFS + // thread's next wake to run the root-seeded walk. The poll below is therefore + // best-effort: it only reconstructs once the representative's getTag() > 0, so the + // retries give the background thread's own post-pass pollWatchedTargets() time to run. // - // Seeding a klass_id's population ring here is not just a ranking shortcut for an - // object some other pass has already tagged - with generations=true, - // ReferenceChainTracker::shouldRunPass() (referenceChains.cpp) gates even this search's - // very first pass on LivenessTracker::hasLeakSignal(), which (once secondsToOOM() has - // nothing to report yet, this early) falls through to selectLeakCandidates() requiring - // consecutive_positive >= LEAK_TREND_HYSTERESIS_BASE for *some* klass, real or not (see - // SEED_EPOCHS_FOR_HYSTERESIS's own comment for why 10 seeded epochs alone cannot clear - // that bar). Seeding SEED_EPOCHS_FOR_HYSTERESIS epochs for this test-chosen id clears it - // outright, which is what actually authorizes the background BFS thread's next - // scheduled wake to run its real, root-seeded runPass() walk - nothing before that point - // has tagged gcRootHolder's own ChainLink yet. pollReferenceChainTargets0() right below - // is therefore best-effort, not guaranteed on this call: it only reconstructs a chain - // once getTag() on the representative is actually > 0 (pollWatchedTargets()'s only - // precondition), which needs that background pass to have already run. The retry loop - // below (a 300ms-sleep-then-dump fallback, repeated across rounds) gives the - // now-unblocked background thread room to do that and have its own post-pass - // pollWatchedTargets() call (threadLoop(), referenceChains.cpp) pick the chain up on its - // own, independent of this method's own poll call succeeding on the first try. + // Seed exactly once: recordKlassPopulationSampleLocked() always appends a ring slot, so + // reseeding would turn the ramp into a sawtooth and destroy the positive slope. // - // Seed exactly once (recordKlassPopulationSampleLocked(), livenessTracker.cpp, always - // *appends* a fresh ring slot rather than overwriting one for a repeated epoch - calling - // this whole block again on a later round would append a second ramp right after the - // first, turning the ring into a non-monotonic sawtooth and destroying the very - // positive-slope signal selectLeakCandidates() needs). Only the poll+dump recheck below - // needs to repeat across rounds - not the seeding - to give the background pass, or a - // lock-contended dump(), more rounds to resolve. - // Representative BEFORE seeding: setKlassPopulationRepresentativeForTest0 - // resolves the representative's real klass id and aliases the synthetic - // id to it, so the hysteresis seeds below land in the real population - // entry (candidate matching is real-id keyed since the leak-tag pool - // redesign; seeds under a bare synthetic id authorize only - // hasLeakSignal()'s candidate-count check and match nothing else). + // Representative before seeding: setting it aliases the synthetic id to the + // representative's real klass id, so the seeds land in the real population entry + // (seeds under a bare synthetic id satisfy only hasLeakSignal()'s candidate count). JavaProfiler.setKlassPopulationRepresentativeForTest0(CHAIN_LINK_TEST_KLASS_ID, gcRootHolder.get(0)); if (!seededTestKlassTrend) { - // Per-(klass, tid) qualification seeds - the persistent allocator - // thread owns every ChainLink allocation, so ITS tid is the - // qualifying one; the seeds accelerate the organic per-tid growth - // its own tracked instances produce every round (see the - // allocator block's own comment for why qualification aging out - // stranded the crawl before this thread existed). + // The allocator thread owns every ChainLink allocation, so its tid is the qualifying + // one; the seeds accelerate the organic per-tid growth it produces every round. int leakTid = (int) allocatorTid.get(); - // Seed counts scale as the organic ramp itself (+1 per epoch), - // NOT some inflated multiple: the ring is one continuous series - // of seeds followed by the real folds, and hasQualifyingGrowth() - // takes the difference of third-window MEANS over that whole - // series. Seeds of epoch*10 (10..150) dwarf the real generation - // counts (the real ramp is +1 per epoch, ~1..15 by test end), so - // the moment real samples displace the seeded tail the slope - // went NEGATIVE (-20..-40 observed live: the "transition cliff") - // and the candidate never returned again. With matched-scale - // seeds the series is [1..15, 1, 2, 3, ...] and every third- - // window pairing across the transition still yields a positive - // slope - the candidate keeps qualifying organically the whole - // run, the real signal seamlessly taking over from the seeds. + // Seed counts match the organic ramp (+1 per epoch): hasQualifyingGrowth() compares + // third-window means over seeds followed by real folds, and inflated seeds (epoch*10) + // made the slope go negative (-20..-40 observed) once real samples displaced them. + // With matched scale ([1..15, 1, 2, 3, ...]) every window pairing stays positive. for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { JavaProfiler.seedKlassPopulationSample0(CHAIN_LINK_TEST_KLASS_ID, epoch, epoch); JavaProfiler.seedTidTrendSample0(CHAIN_LINK_TEST_KLASS_ID, leakTid, epoch, epoch); @@ -538,25 +317,19 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { } if (match == null) { - // A quiet window with no dump() in flight: Profiler::dump() (profiler.cpp) holds every - // entry of _locks[] exclusively for the duration of its own _jfr.dump() call - // (rotateDictsAndRun()'s lockAll()/unlockAll() pair) - the same array - // Profiler::writeReferenceChain() (profiler.cpp) needs a slot from to record an event at - // all. Back-to-back rounds with essentially no gap between one dump() and the next leave - // pollWatchedTargets() (referenceChains.cpp) little to no window to ever win that race; - // sleeping briefly here, then dumping again with no intervening dump()/gc() in between - // (so cleanup_table()'s epoch-advance pass, livenessTracker.cpp, stays a no-op and the - // population trend built up so far is undisturbed), gives it one. + // A quiet window with no dump() in flight: Profiler::dump() holds every _locks[] entry + // exclusively during _jfr.dump() (rotateDictsAndRun()'s lockAll()/unlockAll()), and + // writeReferenceChain() needs a slot to record anything, so back-to-back dumps leave + // pollWatchedTargets() no window. No gc() before this dump, so cleanup_table()'s + // epoch-advance stays a no-op and the trend is undisturbed. Thread.sleep(300); dump(scratchDumpPath); match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); } } - // Grace period: the population trend only became eligible (ring_fill >= 10) partway through - // the loop above, and the same lock contention the loop's own comment describes can still - // delay the resulting write past this method's very last dump() - retry a few more times, - // well past that contention window, before concluding the mechanism genuinely did not fire. + // Grace period: the trend only became eligible partway through the loop, and the same lock + // contention can delay the write past the last dump(). for (int attempt = 0; match == null && attempt < 5; attempt++) { Thread.sleep(1000); dump(scratchDumpPath); @@ -567,18 +340,10 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { "Never observed a datadog.ReferenceChain event whose chain[0] is " + ChainLink.class + " after " + gcRootHolder.size() + " retained ChainLink instances across up to " + totalRounds + " population-growth rounds plus a grace period"); - // chain[0] is the *target* object's own class, not its holder's: - // ReferenceChainTracker::buildChainEvent()'s reconstructChain() (referenceChains.h) - // appends each visited FrontierEntry's referrer_klass leaf-to-root, and referrer_klass is - // populated from the *discovered object's own* GetClassSignature at insertion time (see - // referenceChainJfrRoundtrip_ut.cpp's insert(tag, parent_tag, referrer_klass, depth) calls, - // where tag 3's own referrer_klass is leafKlass, not middleKlass). The target here is - // always some currently-live ChainLink instance - the only class this test's leak-candidate - // population trend was built from - regardless of exactly which of the many instances - // allocated above LivenessTracker::foldKlassCountsLocked() (livenessTracker.cpp) happened to - // keep as its representative. Everything above chain[0] reflects real JDK-internal - // collection representation (e.g. ArrayList's backing array) rather than anything this test - // controls, so it is deliberately not asserted beyond "at least one hop was reconstructed". + // chain[0] is the target's own class, not its holder's: reconstructChain() appends each + // entry's referrer_klass leaf-to-root, and referrer_klass is the discovered object's own + // class (see referenceChainJfrRoundtrip_ut.cpp). Hops above chain[0] are JDK collection + // internals this test does not control, so they are not asserted. assertEquals(ChainLink.class.getName(), match.chain.get(0)); assertTrue(match.targetTag > 0, "targetTag should be a valid, non-zero JVMTI tag"); assertTrue(match.depth >= 0, "depth should be a non-negative hop count"); @@ -586,9 +351,8 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { } finally { gcRootHolder = null; // shared-JVM hygiene - see the static holder's own comment currentBatch = null; - // Shut the persistent allocator thread down before leaving: this is a - // shared, no-forkEvery test JVM, and a leftover allocation thread would - // keep allocating (and keep its tid trend alive) into every later test. + // A leftover allocator thread would keep its tid trend alive into later tests in this + // shared, no-forkEvery JVM. allocator.interrupt(); allocator.join(1000); Files.deleteIfExists(scratchDumpPath); @@ -596,89 +360,42 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { } /** - * A second, independent end-to-end scenario for the same target-selection mechanism as - * {@link #shouldReconstructReferrerChainToGcRoot()}, using a shape much closer to a real-world - * leak than that method's hand-rolled {@link ChainLink} list: an ever-growing, never-evicted - * {@link java.util.HashMap}-backed cache - a very common real leak pattern - holding - * {@link CachedPayload} values. Beyond confirming a chain fires for the leaking value type, - * this also asserts the reconstructed chain actually threads back through - * {@code java.util.HashMap}'s own internal storage ({@code HashMap$Node}), i.e. that the - * mechanism correctly walks a real JDK collection's internals rather than only ever having - * been proven against a single purpose-built linked fixture. + * Same target-selection mechanism as {@link #shouldReconstructReferrerChainToGcRoot()}, against + * a common real-world leak shape: an ever-growing, never-evicted {@link java.util.HashMap} cache + * of {@link CachedPayload} values. Also asserts the chain threads through HashMap's internal + * storage, proving the walk handles real JDK collection internals. * - *

    Why {@code cache} is a local variable, not a {@code static} field (found the hard - * way): {@code ReferenceChainTracker::expandFrontier()} (referenceChains.cpp) freezes - * whatever children it observes for a node the moment that node is expanded - it never - * re-examines an already-{@code EXPANDED} entry for newly-added children, regardless of how - * many times root/stack-ref enumeration itself reruns over the search's lifetime - * ({@code runPassManualWalk()}'s own comment). A {@code static} field becomes a root at - * class-load time, essentially guaranteeing some early pass catches it (and marks it - * {@code EXPANDED}) while still empty - permanently blocking discovery of anything added to it - * afterward, no matter how many rounds run. A local variable created fresh at the top of this - * method, immediately followed by round-1 allocation, gives root enumeration a real chance to - * catch it only after it already holds live data - mirroring - * {@link #shouldReconstructReferrerChainToGcRoot()}'s {@code gcRootHolder}, which works for - * exactly this reason (a local {@code ArrayList}, not a {@code static} one). - * - *

    See this class's own header comment for why this runs as {@code @Order(2)}, before - * {@link #shouldReportAbandonedSearchOnTinyFrontierCap()}, and for why its relative order - * against {@link #shouldReconstructReferrerChainToGcRoot()} does not itself matter. + *

    Why {@code cache} is a local variable, not a {@code static} field: + * {@code expandFrontier()} freezes a node's children when it is expanded and never re-examines + * it. A {@code static} field is a root from class-load time, so an early pass would almost + * certainly expand it while still empty and block discovery of everything added later. */ @Test @Order(2) public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exception { - // Seed cache with one live entry *before* the resets below, mirroring - // shouldReconstructReferrerChainToGcRoot()'s own gcRootHolder seeding (see that method's - // comment): HashMap's backing table also starts out as a shared, lazily-replaced empty - // sentinel (only allocated on the first put()), so the same one-shot-BFS-freeze hazard - // documented there - a concurrent GC-triggered walk expanding this node's children while the - // backing table is still the empty sentinel, permanently freezing an empty children set - - // applies here too. Seeding one entry first means the backing table is never empty at any - // point after the search (re)starts. + // Seed one entry before the resets, for the same empty-sentinel/frozen-children reason as + // gcRootHolder's seeding: HashMap's table is only allocated on the first put(). Map cache = new HashMap<>(); cache.put("cache-leak-seed", new CachedPayload("cache-leak-seed")); if ("debug".equals(System.getProperty("ddprof_test.config"))) { - // Mirrors shouldReconstructReferrerChainToGcRoot()'s own reset (see that method's comment): - // the natural restartSearch() cycle (shouldRunPass(), referenceChains.cpp) that would - // otherwise give this method its own fresh root walk once shouldReconstructReferrerChainToGcRoot()'s - // own ChainLink candidate is found and its tags released is gated on cadence/pacing budget - // (canAffordNewSearch()) - not guaranteed to fire again before this method's own round/retry - // budget runs out. Force a genuine fresh search here too, rather than depend on that timing, - // so cache (below) is guaranteed reachable by a real walk regardless of it. Debug-only: this - // native seam does not exist in a release build. - // - // resetReferenceChainSearchForTest0() only resets ReferenceChainTracker's own search state - // (frontier/tags/search-progress fields); it does not touch LivenessTracker's per-klass - // population-history rings, which selectLeakCandidates() consults independently to decide - // which klass is "trending". Without also resetting those, a klass ChainLink already - // accumulated a positive slope for during shouldReconstructReferrerChainToGcRoot() can still - // outrank CachedPayload as the leak candidate here, so the fresh search below ends up - // reconstructing ChainLink's chain again instead of CachedPayload's. + // The natural restart after the @Order(1) search ends is gated on canAffordNewSearch() and + // not guaranteed within this method's retry budget, so force a fresh search. The population + // reset is also needed: resetReferenceChainSearchForTest0() leaves LivenessTracker's rings + // alone, and ChainLink's leftover positive slope could outrank CachedPayload. + // Debug-only: these native seams do not exist in a release build. JavaProfiler.resetKlassPopulationForTest0(); JavaProfiler.resetReferenceChainSearchForTest0(); } Path scratchDumpPath = Paths.get("referencechains-cache-leak-scratch.jfr"); try { - // Same round-growth/retry shape as shouldReconstructReferrerChainToGcRoot() - see that - // method's own comment for why the loop self-adjusts rather than committing to a fixed - // round count, why per-round scale is sized against ObjectSampler's real 256KiB sampling - // floor rather than this test's own "memory=64" request, why totalRounds is capped at - // 16 rather than a larger margin above the 10-round minimum (shared-fork heap headroom), - // and why per-round growth itself is clamped to round 10 (Math.min(round, 10) below). + // Same round-growth/retry shape (and caps) as shouldReconstructReferrerChainToGcRoot(). ReferenceChainAssertions.JfrChainMatch match = null; boolean seededTestKlassTrend = false; int totalRounds = 16; - // Pre-generate every key this loop will ever need, up front, rather than concatenating a - // fresh String on every put() below. CachedPayload's own class comment already documents - // why this fixture uses plain long fields instead of a byte[] (an earlier version's byte[] - // field stole every allocation-sampling hit from CachedPayload itself) - the key String - // (plus its own backing byte[]) is the same kind of same-instant, size-weighted-sampler - // competitor, just a *separate* object instead of a field. Generating all keys in one - // batch before the round loop starts lets their own klass_population trend flatten out - // (selectLeakCandidates() needs KLASS_POPULATION_MIN_FILL_FOR_TREND=10 *new* samples, not - // just a nonzero ring) well before CachedPayload's own ring starts growing, so the sampler - // has nothing else in flight to compete with once round 1 begins. + // Keys are pre-generated: a String key (and its byte[]) allocated alongside each + // CachedPayload would compete for the size-weighted sampler. Allocating them all up front + // lets their own population trend flatten before CachedPayload's starts growing. int maxEntries = 0; for (int round = 1; round <= totalRounds; round++) { maxEntries += Math.min(round, 10) * 600; @@ -690,12 +407,8 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc int nextKey = 0; for (int round = 1; round <= totalRounds && match == null; round++) { - // Matches shouldReconstructReferrerChainToGcRoot()'s own *600 growth rate (not *300, this - // method's previous value) - CachedPayload and ChainLink are near-identical in size (same - // padding fields; CachedPayload is actually slightly smaller, missing ChainLink's `next` - // reference), so halving the growth rate here bought no headroom and instead just let - // CachedPayload's own population ring stall short of KLASS_POPULATION_MIN_FILL_FOR_TREND - // within the same totalRounds budget both methods share. + // Same *600 rate as the ChainLink test: the two classes are near-identical in size, and a + // lower rate let CachedPayload's ring stall short of KLASS_POPULATION_MIN_FILL_FOR_TREND. int newEntries = Math.min(round, 10) * 600; int keyOffset = nextKey; Thread allocator = new Thread(() -> { @@ -713,21 +426,10 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc match = ReferenceChainAssertions.findMatchForClass(events1, CachedPayload.class); if (match == null && "debug".equals(System.getProperty("ddprof_test.config"))) { - // Same deterministic short-circuit as shouldReconstructReferrerChainToGcRoot() - see - // that method's own comment for the full mechanism (why seeding - // SEED_EPOCHS_FOR_HYSTERESIS epochs is what actually authorizes the background BFS - // thread's next real pass, and why this method's own pollReferenceChainTargets0() call - // right below is therefore best-effort rather than something the seed alone - // guarantees on this call). Seed exactly once - see that method's own comment for why - // reseeding the same ramp on a later round would corrupt the ring into a non-monotonic - // sawtooth and destroy the positive-slope signal instead of just re-establishing it. - // Representative BEFORE seeding - same real-id aliasing rationale as - // shouldReconstructReferrerChainToGcRoot()'s own comment above. + // Same seed-once, representative-first, best-effort-poll rationale as + // shouldReconstructReferrerChainToGcRoot(). JavaProfiler.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get(keys[0])); if (!seededTestKlassTrend) { - // Per-(klass, tid) qualification seeds - see LeakingCacheScenario's - // own seeding block for why this thread's tid (the representative's - // creator) is the qualifying one for the one-cohort-per-round shape. int leakTid = JavaProfiler.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { JavaProfiler.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); @@ -787,94 +489,37 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc } /** - * This test's stated abandonment-path scenario: an artificially tiny frontier cap - * ({@code getProfilerCommand()}'s {@code framecap=1} for this test method, see that method's own - * comment for why {@code budget} must stay larger than {@code framecap}) makes the very first - * BFS pass hit the frontier cap once a second root-referenced object is discovered, so - * {@code runPass()} (referenceChains.cpp) abandons the search rather than silently truncating it - * - or, if this method's own - * {@code framecap=1} lost the race to size the shared {@code FrontierTable} (see - * {@code getProfilerCommand()}'s own comment on this method), its {@code ttl=100} fallback - * abandons the search instead once enough wall-clock time has passed, which by the time this - * method runs it always has. Either path exercises the same {@code buildAbandonedEvent()} / - * {@code Profiler::writeReferenceChainAbandoned()} / {@code FlightRecorder::recordReferenceChainAbandoned()} - * write into the recording that {@code Profiler::dump()} triggers - this method only asserts that - * a {@code datadog.ReferenceChainAbandoned} event exists, not which reason produced it. + * Drives the shared search to {@code SearchState::ABANDONED} - via {@code framecap=1} when this + * method sizes the {@code FrontierTable}, otherwise via the {@code ttl=100} fallback (see + * {@code getProfilerCommand()}) - and asserts a {@code datadog.ReferenceChainAbandoned} event is + * written on {@code dump()}, without asserting which reason produced it. * - *

    {@code @Order(3)}: this permanently exhausts the process-wide {@code ReferenceChainTracker} - * singleton's one-and-only search (see this class's header comment), so it must run after - * {@link #shouldReconstructReferrerChainToGcRoot()} and - * {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}. It used to rely on being - * the last method in source order plus JUnit's default (unannotated methods sort after - * {@code @Order}-annotated ones) for that - which happened to guarantee run order, but not the - * search *state* this method's own comment above presumes: if either earlier test's own - * organic GC/allocation-sampling trend never fired in time (observed in practice - real - * flakiness, not this test's fault), the shared search can still be {@code RUNNING} with a - * large, non-fresh frontier when this method starts, instead of the small, just-abandoned one - * its own {@code framecap=1}/{@code ttl=100} scenario assumes. Calling - * {@code resetReferenceChainSearchForTest0()} here, exactly as both earlier tests already do for - * their own scenarios, makes this method's own precondition (a fresh search) something it - * establishes itself rather than something it presumes a prior test left behind. Debug-only: - * this native seam does not exist in a release build. + *

    {@code @Order(3)}: abandoning the singleton search would break the two success-path tests. + * The debug-build reset makes the fresh-search precondition explicit: if an earlier test's + * trend never fired, the search may still be {@code RUNNING} with a large frontier. * - *

    Why this method also seeds a klass population trend, despite requesting no - * {@code generations=true} of its own (found the hard way: {@code resetReferenceChainSearchForTest0()} - * alone was not enough - the search's first pass never ran at all, so it could never reach - * {@code SearchState::ABANDONED} for {@code dump()} to observe): {@code LivenessTracker::_gc_generations} - * (livenessTracker.cpp's {@code initialize()}) is only ever updated when a {@code start()} command - * actually requests liveness tracking. This method's own command now requests - * {@code memory=64:l:1.0} (required for the tracker to start at all, see - * {@code getProfilerCommand()}), so {@code LivenessTracker::initialize()} resets - * {@code gcGenerationsEnabled()} to false and {@code ReferenceChainTracker::hasLeakSignal()} - * returns true unconditionally. The seeding below is kept as defensive insurance in case a - * future command change re-enables generations (then {@code hasLeakSignal()}, which - * {@code shouldRunPass()} consults for even this search's very first pass, falls through to - * {@code selectLeakCandidates()} exactly as the two tests above do - with nothing left over from - * {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}'s own - * {@code resetKlassPopulationForTest0()} call to satisfy it), and is harmless otherwise. Seeding - * {@code SEED_EPOCHS_FOR_HYSTERESIS} epochs for a dedicated klass id (see - * {@code SEED_EPOCHS_FOR_HYSTERESIS}'s own comment for the full derivation) clears that gate the - * same way it does there - this method never sets a representative or polls for it, since it has - * no reconstructed chain of its own to prove; {@code selectLeakCandidates()} - * (livenessTracker.cpp) counts a qualifying entry toward {@code hasLeakSignal()}'s candidate - * count regardless of whether its representative was ever set, and - * {@code resolveCandidateRepresentative()} is null-safe against one that wasn't. + *

    Why the population seeding: {@code resetReferenceChainSearchForTest0()} alone was + * observed never running a first pass. With this command ({@code memory=64:l:1.0}, no + * {@code generations=true}) {@code LivenessTracker::initialize()} clears generations, so + * {@code hasLeakSignal()} returns true unconditionally; the seeding is defensive in case the + * command re-enables generations, in which case {@code shouldRunPass()} falls through to + * {@code selectLeakCandidates()} with nothing left from the cache test's population reset. No + * representative is set: a qualifying entry counts toward {@code hasLeakSignal()} regardless, + * and {@code resolveCandidateRepresentative()} is null-safe. * - *

    Why the seeding is preceded by a {@code System.gc()} and a throwaway {@code dump()} - * (found the hard way, twice: a bare {@code dump()} with no preceding GC still lost the - * race): {@code cleanup_table()} (livenessTracker.cpp) compares - * {@code Profiler::classMap()->generation()} against its own cached copy and, on a mismatch - - * which this method's own fresh {@code start()} causes, by clearing that map - wipes - * {@code _klass_population} back to empty before folding anything else, exactly once per - * mismatch; but that comparison sits *after* an early-return guard - * (`{@code if (!is_epoch_owner && !forced) return;}`) that a {@code dump()}-triggered call - * (always {@code forced=false}) only clears when a genuinely new, not-yet-processed GC epoch - * is pending - a {@code dump()} with no GC since the last {@code cleanup_table()} pass never - * reaches the mismatch check at all. The two methods above never hit any of this because their - * own seed loop runs only after their round-1 {@code System.gc()}+{@code dump()} pair has - * already triggered that one-time wipe-and-resync as a side effect of their own allocation - * loop; seeding before any GC/dump of this method's own would instead leave the seed to be - * discarded by the *next* call that does clear the guard - this class's own - * {@code maybeForceCleanup()} tick (called once per {@code ReferenceChainTracker::threadLoop()} - * wake, {@code forced=true} unconditionally clears the guard), which fires on this method's own - * 3 {@code System.gc()} calls below regardless. The explicit {@code System.gc()}+{@code dump()} - * pair here forces that one-time wipe-and-resync to happen now, before seeding, exactly like - * the two methods above get for free from their own round-1 allocation loop. + *

    Why a {@code System.gc()} + throwaway {@code dump()} precede the seeding: + * {@code cleanup_table()} wipes {@code _klass_population} once when the class map's + * generation changes (which this method's fresh {@code start()} causes), but only after the + * {@code if (!is_epoch_owner && !forced) return;} guard, which an unforced dump-triggered call + * passes only when a new GC epoch is pending. Without a GC first, the wipe would be deferred to + * the next forced cleanup (on the BFS thread's wake after the GCs below) and discard the seed. + * The success-path tests get this for free from their round-1 GC + dump. */ @Test @Order(3) public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { if ("debug".equals(System.getProperty("ddprof_test.config"))) { - // See this method's own javadoc ("Why this method also seeds a klass population trend...") - // for why this is kept even though this method requests no generations=true of its own. - // - // A System.gc() + throwaway dump() first - see this method's own javadoc ("Why the seeding - // is preceded by...") for why *both* are needed: a dump() alone triggers cleanup_table() - // with forced=false, which only clears the class-map-generation-mismatch guard (and - // therefore only wipes/resyncs _klass_population) when a new GC epoch is actually pending - - // this System.gc() is what makes that true for this call, rather than leaving it to chance - // (or to this method's own 3 System.gc() calls below, which run after the seed and would - // let maybeForceCleanup()'s forced=true tick discard it first). + // GC + throwaway dump before seeding - see javadoc. System.gc(); Path warmupDumpPath = Paths.get("referencechains-abandoned-test-warmup.jfr"); try { @@ -883,10 +528,7 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { Files.deleteIfExists(warmupDumpPath); } JavaProfiler.resetKlassPopulationForTest0(); - // Per-(klass, tid) qualification seeds - gate-only usage (this test - // drives the search to ABANDONED and asserts on the abandon marker, no - // tagged instance needs to match), so the tid is this thread's real one - // but any distinct tid would do. + // Gate-only seeding: no tagged instance needs to match, so any tid would do. int abandonTid = JavaProfiler.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { JavaProfiler.seedKlassPopulationSample0(ABANDON_TEST_KLASS_ID, epoch * 10, epoch); @@ -911,17 +553,9 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { Path dumpPath = Paths.get("referencechains-abandoned-test.jfr"); try { - // Even with the seeding above clearing hasLeakSignal()'s gate so the search's first pass is - // actually allowed to start (this method's own javadoc), that pass still has to be woken by - // the BFS thread and still has to actually reach the frontier cap/ttl cutoff, inside whatever - // real wall-clock time this shared, no-forkEvery test JVM's own scheduling happens to grant - // it (this class's own header comment) - a single fixed-length sleep-then-dump-then-assert, - // with no room to retry, previously failed outright whenever that one attempt landed a beat - // early. Retry across a few more dumps, spaced far enough apart to tolerate a slow - // BFS-thread wake or a walk over a larger-than-usual inherited heap, before concluding the - // search genuinely never abandoned - mirroring the bounded retry loops - // shouldReconstructReferrerChainToGcRoot()/shouldReconstructReferrerChainThroughUnboundedCacheLeak() - // already use above. + // The pass still has to be woken and reach the frontier-cap/ttl cutoff within whatever time + // the shared JVM grants it; a single fixed sleep-then-assert failed when it landed a beat + // early. Retry spaced far enough apart to tolerate a slow wake or a larger inherited heap. JfrEvents abandoned = null; for (int attempt = 0; attempt < 8; attempt++) { dump(dumpPath); @@ -941,15 +575,9 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { } /** - * Referrer-type fixture shared by both the success-path and abandonment-path tests. The 32 - * {@code long} fields below exist purely so {@link #shouldReconstructReferrerChainToGcRoot()} - * needs far fewer instances to allocate a given number of megabytes of ChainLink - deliberately - * plain fields, not a nested array: an array field would be a *second*, separate heap - * allocation, and the allocation-sampling interval that method's comment describes picks - * whichever allocation happens to cross its byte threshold size-weighted, so a same-instance - * companion array would take sampling attention away from ChainLink itself rather than adding - * to it. Plays no role in {@link #shouldReportAbandonedSearchOnTinyFrontierCap()}'s tiny, - * two-object fixture beyond trivially increasing its size. + * Referrer-type fixture for the success-path and abandonment-path tests. The 32 {@code long} + * fields cut the instance count needed per sampled megabyte. Plain fields, not an array: an + * array would be a separate allocation competing for the size-weighted sampler. */ private static final class ChainLink { final String name; @@ -968,17 +596,9 @@ private static final class ChainLink { } /** - * Realistic-leak fixture for {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}: - * models the single most common real-world leak shape this mechanism is meant to catch - an - * unbounded, never-evicted cache - as opposed to {@link ChainLink}'s hand-rolled linked list. - * The 32 {@code long} fields exist purely so a given number of megabytes needs far fewer - * entries to reach {@code ObjectSampler}'s real 256KiB sampling floor - deliberately plain - * fields, not a nested array, for exactly the reason {@link ChainLink}'s own comment already - * documents: a same-instance companion array would be a *second*, separate heap allocation - * that the size-weighted allocation sampler would compete for, taking sampling attention away - * from {@code CachedPayload} itself (an earlier version of this fixture used a {@code byte[]} - * field and never observed a single {@code CachedPayload} sample as a result - the sampler was - * catching the array instead). + * Cache-value fixture for {@link #shouldReconstructReferrerChainThroughUnboundedCacheLeak()}. + * Padded with plain {@code long} fields for the same reason as {@link ChainLink}; a + * {@code byte[]} field was observed taking every sample away from {@code CachedPayload}. */ private static final class CachedPayload { final String key; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java index c87b5bda2a..982864ba03 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java @@ -69,20 +69,15 @@ private StaticFieldGrowingCollectionScenario() {} private static final int TOTAL_ROUNDS = 25; - // Tail attempts (below) rotate the watched chunk they append to LEAK_BUFFER, so the backing - // ArrayList's pre-sized capacity must cover them too - see LEAK_BUFFER_CAPACITY's own comment - // for why elementData must never reallocate. + // Each tail rotation appends a chunk to LEAK_BUFFER, so LEAK_BUFFER_CAPACITY covers these too. private static final int TAIL_ATTEMPTS = 40; - // How many consecutive tail attempts the SAME freshly-appended chunk stays the watched target - // before rotating to a new one - see the tail loop's own comment for why a chunk must be given - // several attempts (its discovery matures across walk cycles, slower than one 1.5s attempt) - // and why a chunk that still has not resolved is assumed record-dead and abandoned. + // Consecutive tail attempts the same chunk stays the watched target before rotating - see the + // tail loop. private static final int TAIL_CHUNK_ATTEMPTS = 8; - // Seeds + lateChunk + one per round + one per tail rotation (sized for the worst case of one - // per attempt so the budgets above can be retuned without touching this): - // LEAK_BUFFER's elementData never reallocates. + // Seeds + lateChunk + one per round + one per tail attempt (worst case), so LEAK_BUFFER's + // elementData never reallocates. private static final int LEAK_BUFFER_CAPACITY = SEED_CHUNKS + 1 + TOTAL_ROUNDS + TAIL_ATTEMPTS; @@ -129,8 +124,6 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD && ("debug".equals(testConfig) || testConfig.endsWith("san")); System.out.println(SEAM_PATH_MARKER + debugBuild); int hysteresisEpoch = 0; - // Declared at method scope: the per-round maintenance seeding below (in - // its own debugBuild block) reuses this tid. int leakTid = 0; // Snapshotted BEFORE seeding anything, while the BFS thread's search has not run a single // pass yet (referenceChainPassesRunForTest0() reads 0 here) - see the wait loop below for why @@ -173,19 +166,11 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // representative ref stays live. JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, LEAK_BUFFER.get(0)); - // Seed enough hysteresis-qualifying epochs to unlock ReferenceChainTracker::hasLeakSignal()/ - // shouldRunPass() - AFTER the representative above (set-representative-first is required, - // see that call's own comment). hasLeakSignal() only asks selectLeakCandidates() for a - // nonzero count - - // it never reads/dereferences the representative field - so this step's only job is - // authorizing the BFS thread's first pass to actually start. Doing this BEFORE waiting on - // passesRun() advancing below is load-bearing, found the hard way: a pass only ever runs - // once shouldRunPass() sees a leak signal - waiting on passesRun() before seeding anything is - // a deadlock, not just a slow path. - // Per-(klass, tid) qualification seeds (see LeakTagCorrelationScenario's own - // seeding block for the rationale): the qualifying tid must be the thread that - // allocates the tracked instances - this thread allocates every LEAK_BUFFER - // chunk, including the lateChunk this scenario watches. + // Seed hysteresis-qualifying epochs so hasLeakSignal()/shouldRunPass() authorize the BFS + // thread's first pass. Must precede the passesRun() wait below: a pass only runs once + // shouldRunPass() sees a leak signal, so waiting first deadlocks. Per-(klass, tid) seeds + // (see LeakTagCorrelationScenario): the qualifying tid must be the allocating thread, and + // this thread allocates every LEAK_BUFFER chunk. leakTid = JavaProfiler.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { JavaProfiler.seedKlassPopulationSample0(LEAK_BUFFER_TEST_KLASS_ID, epoch * 10, epoch); @@ -199,14 +184,9 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // entire point of this scenario: proving that an element appended to a static field's // collection AFTER that field's List is already EXPANDED gets discovered via a LATER // re-expansion of that stale entry, not via the very first pass that admitted it. - // admitStaticFieldRoots() and expandFrontier() are two steps of the SAME runPass() call, and - // neither the static-field sweep's completion nor a changed pass counter proves the holders - // were expanded (see the debug branch below), so the wait reads their frontier state directly. - // Release (non-DEBUG) builds have no such seam and fall back to a generous sleep - acceptable - // there because the non-debug path never pins a specific representative anyway (see the - // debugBuild branch below): it just waits on real, much-slower allocation sampling to notice - // the growth on its own, so a few extra seconds of margin either way changes nothing about - // what the eventual match would prove. + // Release (non-DEBUG) builds have no frontier-state seam and fall back to a sleep: they never + // pin a specific representative and rely on much-slower real allocation sampling, so the + // margin changes nothing about what the eventual match proves. if (debugBuild) { // A changed pass counter alone is not proof that LEAK_BUFFER's holders were expanded: // truncated passes also bump it, and admitStaticFieldRoots() only processes a bounded @@ -282,14 +262,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // per round instead is self-healing exactly the way the real leak this scenario mirrors is: // LEAK_BUFFER keeps growing every round regardless of any one-time startup housekeeping, so // however many rounds a wipe costs, the trend simply resumes accumulating from the next - // round's seed call. (Waiting for the first pass to fully complete above already makes this - // particular race very unlikely to matter by this point, but there is no reason to give up the - // self-healing margin now that it costs nothing.) + // round's seed call. ReferenceChainAssertions.ChainMatch match = null; for (int round = 1; round <= TOTAL_ROUNDS && match == null; round++) { - // Continues growing LEAK_BUFFER every round after lateChunk - mirrors the real leak's own - // continuous growth - but lateChunk itself (created above, strictly after the first sweep) - // stays the fixed target this scenario watches for throughout. + // Mirrors the real leak's continuous growth; lateChunk stays the watched target. LEAK_BUFFER.add(new byte[CHUNK_BYTES]); System.gc(); @@ -305,13 +281,9 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // legitimately produces a continuous real signal in production; seeding continuously here // is what actually mirrors that, not an artifact to gate away. // - // Same debug-only seeded-representative short-circuit as LeakingCacheScenario's own - - // watches lateChunk specifically (added after the static field's first sweep), not - // whatever the real allocation sampler happens to pick. See SEED_EPOCHS_FOR_HYSTERESIS's - // own comment for the minimum epoch count this needs - kept uncapped here (not stopped - // once that minimum is reached) specifically so a wipe that costs an early round or two - // still leaves enough remaining rounds to reach it, rather than exhausting a fixed budget - // of calls before ever getting there. + // The representative pins lateChunk, not whatever the real sampler picks. Seeding stays + // uncapped past SEED_EPOCHS_FOR_HYSTERESIS so a wipe costing early rounds still leaves + // enough rounds to requalify. hysteresisEpoch++; JavaProfiler.seedKlassPopulationSample0( LEAK_BUFFER_TEST_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); @@ -357,20 +329,13 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } // A cold external JVM's one-shot root-seeded walk over its whole reachable graph (full - // JUnit/JMC/Gradle-worker classpath) takes far longer than this scenario's own round loop - - // found the hard way: a first attempt at this scenario's tail wait (5 x 1s) gave up and - // printed NOT_FOUND_MARKER mere moments before the tracker's own runPass() actually - // completed (searchState=COMPLETED) and successfully built the chain. Mirrors - // ExternalProcessReferenceChainTest's own comment on why budget/pausetarget are raised this - // far for exactly this reason. + // JUnit/JMC/Gradle-worker classpath) takes far longer than the round loop - a 5 x 1s tail + // gave up moments before runPass() completed and built the chain. int watchedIndex = -1; int attemptsOnWatchedChunk = 0; for (int attempt = 0; match == null && attempt < TAIL_ATTEMPTS; attempt++) { Thread.sleep(1500); - // Same passesRun()-advanced check as the round loop above (-DDEBUG builds only: - // debug/asan/tsan) - by this point in a real run it has virtually always already flipped, - // but there is no reason to drop the guarantee here just because the round loop above - // didn't need it. + // Same passesRun()-changed guard as the round loop. long requiredTargetTag = 0; if (debugBuild) { if (JavaProfiler.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java index f72d4fa1ec..4aecb0b906 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java @@ -33,16 +33,12 @@ * {@code ThreadLocal} map must still produce a correlated * {@code datadog.ReferenceChain} - {@code targetTag} in the leak-tag pool * range, matched by a {@code datadog.HeapLiveObject} {@code leakTag}. The - * scenario is {@code ThreadLocalLeakScenario}; this test drives it and - * asserts the reported correlation, with a filtered child-log tail embedded - * in any failure message for diagnosability without a redeploy. + * scenario is {@code ThreadLocalLeakScenario}. */ @Tag("slow") public class ThreadLocalLeakReferenceChainTest extends AbstractProcessProfilerTest { - // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE) - the - // contract under test, asserted against the runtime event values the child - // reports. + // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE). private static final long LEAK_TAG_BASE = 0x40000000L; private static final long LEAK_TAG_POOL_SIZE = 256; @@ -177,13 +173,7 @@ void shouldCorrelateLeakOnThreadCreatedAfterRecordingStart() throws Exception { } } - /** - * A filtered summary of the child's TEST_LOG stream for failure messages - - * same first-things-to-check selection as - * LeakTagCorrelationReferenceChainTest.diagnostics(): the thread-walk and - * pass summaries (the new machinery this scenario exercises), the gotw - * batch-size trace, and any leak-tag interception or correlation lines. - */ + /** Filtered summary of the child's TEST_LOG stream, embedded in failure messages. */ private static String diagnostics(Deque testLogTail) { if (testLogTail.isEmpty()) { return "\n(no TEST_LOG output captured - non-debug build or no reference-chain passes ran)"; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java index f9894eea9a..0d5c0a5ccb 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -52,10 +52,9 @@ public final class ThreadLocalLeakScenario { private ThreadLocalLeakScenario() {} /** - * The leak sink: a per-thread growing list, touched ONLY by the leaking - * thread below. Deliberately NOT static-retained anywhere - the whole - * point is that the only durable path to the chunks runs through the - * leaking thread's ThreadLocalMap. + * The leak sink, touched ONLY by the leaking thread. Deliberately NOT + * static-retained: the only durable path to the chunks must run through + * the leaking thread's ThreadLocalMap. */ private static final ThreadLocal> SINK = ThreadLocal.withInitial(() -> new ArrayList<>()); @@ -108,11 +107,9 @@ byte[] take() { } /** - * Bounded wait for the leak thread's first publish. Required in both - * thread-start modes: run() consumes the handoff right after the - * profiler starts, and neither starting the thread before - * {@code profiler.execute()} nor after it guarantees the thread has - * allocated its seed chunks and published by then. Must be called + * Required in both thread-start modes: neither starting the thread + * before {@code profiler.execute()} nor after it guarantees it has + * published by the time run() consumes the handoff. Must be called * before take() (it polls seedChunk, which take() clears). */ boolean awaitPublication(long timeoutMs) throws InterruptedException { @@ -211,10 +208,6 @@ public static void run(JavaProfiler profiler, String startCommand, if (lateThread) { leakThread.start(); } - // The debug seeding below consumes the handoff (rep chunk + tid) - // immediately. Neither thread-start mode guarantees the leak thread has - // published by now (profiler.execute() returning says nothing about the - // leak thread's progress), so wait for its first publish in both. if (!handoff.awaitPublication(30000)) { System.out.println(NOT_FOUND_MARKER); return; @@ -223,11 +216,9 @@ public static void run(JavaProfiler profiler, String startCommand, boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); int leakTid = 0; if (debugBuild) { - // Representative first, seeds second - same aliasing rationale as - // LeakTagCorrelationScenario (the rep's real klass id must own the - // seeds or the seeds authorize nothing). The rep chunk stays durably - // reachable only through the leak thread's ThreadLocalMap - the - // take() above dropped this handoff's reference. + // Representative first, seeds second: the rep's real klass id must + // own the seeds or they authorize nothing. take() drops the handoff's + // reference, so the rep chunk stays reachable only via ThreadLocalMap. JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, handoff.take()); leakTid = handoff.tid; for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { @@ -317,7 +308,6 @@ public static void run(JavaProfiler profiler, String startCommand, System.out.println(FOUND_MARKER + best.payloadChain.targetTag); } - /** True if any hop of the leaf-to-root chain is the ThreadLocalMap (or one of its nested types). */ private static boolean passesThroughThreadLocalMap(Object[] rawChain) { for (Object element : rawChain) { if (element instanceof IMCType) { @@ -341,14 +331,11 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { } /** - * Scans one dump: the first leaked {@code byte[]} chain (if any) that runs - * through a ThreadLocalMap hop - any other byte[] chain does not prove the - * thread-scoped walk and is ignored - whether - * its targetTag is in the leak-tag pool, and how many - * {@code datadog.HeapLiveObject} events for a leaked {@code byte[]} carry - * that same tag in {@code leakTag}. Simplified from - * LeakTagCorrelationScenario.scan() - no transient/noise gate scans (those - * are that scenario's own contracts). + * Scans one dump for a leaked {@code byte[]} chain through a ThreadLocalMap + * hop, whether its targetTag is in the leak-tag pool, and how many + * {@code datadog.HeapLiveObject} {@code byte[]} events carry that tag in + * {@code leakTag}. No transient/noise gate scans - those are + * LeakTagCorrelationScenario's contracts. */ private static ScanResult scan(Path scratchDumpPath) throws Exception { if (!Files.exists(scratchDumpPath)) { From 9042c9054173cb06779a78e867cc1aaa1894c2bf Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 16 Oct 2026 16:49:37 +0200 Subject: [PATCH 14/28] Address review: serialize leak-tag incarnation, evict dead-owner chains - Serialize leakTagIncarnation into HeapLiveObject and ReferenceChain so consumers join on (tag, stamp) across chunks; tags alone are recycled - Erase chains whose leak-tag owner died during drain and reject them in cacheResolvedChain: a stamp mismatch can never become valid again - Revert test seams to package-private; forward through ReferenceChainTestHooks in ddprof-test; restore the not-read-only warning on shouldRunPassForTest0 - Count drained chains per leaf class (drainReferenceChainEventCountForLeaf0) - Accept debug and *san builds for test-hook gating (asan/tsan nightlies) - Fix cache-leak test flake: persistent allocator thread, matched-scale seeds - Verify owner identity in leak-tag correlation assertions - Process-test hardening: bounded waits, synchronized log tail, child timeout budget, destroy on interrupt; copyright header --- ddprof-lib/src/main/cpp/event.h | 13 +- ddprof-lib/src/main/cpp/flightRecorder.cpp | 13 +- ddprof-lib/src/main/cpp/flightRecorder.h | 3 +- ddprof-lib/src/main/cpp/javaApi.cpp | 46 +++- ddprof-lib/src/main/cpp/jfrMetadata.cpp | 4 + ddprof-lib/src/main/cpp/livenessTracker.cpp | 3 + .../src/main/cpp/referenceChainEvents.cpp | 26 +- ddprof-lib/src/main/cpp/referenceChains.h | 15 +- .../com/datadoghq/profiler/JavaProfiler.java | 57 +++-- .../src/test/cpp/livenessTracker_ut.cpp | 64 +++++ .../cpp/referenceChainJfrRoundtrip_ut.cpp | 17 +- .../test/cpp/referenceChainsTrackerTests.inc | 22 +- .../profiler/AbstractProcessProfilerTest.java | 31 ++- .../datadoghq/profiler/ExternalLauncher.java | 77 +++--- .../profiler/JavaProfilerApiSurfaceTest.java | 19 ++ .../profiler/ReferenceChainTestHooks.java | 105 ++++++++ .../AggressiveLeakReferenceChainTest.java | 48 ++-- .../ExternalProcessReferenceChainTest.java | 4 +- .../LeakTagCorrelationReferenceChainTest.java | 82 +++--- .../LeakTagCorrelationScenario.java | 237 ++++++++++++++---- .../referencechains/LeakingCacheScenario.java | 13 +- .../referencechains/PersistentAllocator.java | 131 ++++++++++ .../ReferenceChainAssertions.java | 20 ++ .../ReferenceChainJfrParserTest.java | 35 ++- .../ReferenceChainTestSeamsTest.java | 51 ++-- .../ReferenceChainTrackingTest.java | 118 ++++----- .../StaticFieldGrowingCollectionScenario.java | 53 ++-- .../ThreadLocalLeakReferenceChainTest.java | 46 ++-- .../ThreadLocalLeakScenario.java | 54 ++-- 29 files changed, 1007 insertions(+), 400 deletions(-) create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java diff --git a/ddprof-lib/src/main/cpp/event.h b/ddprof-lib/src/main/cpp/event.h index d87f40d672..1e8dc91b5e 100644 --- a/ddprof-lib/src/main/cpp/event.h +++ b/ddprof-lib/src/main/cpp/event.h @@ -94,6 +94,11 @@ class ObjectLivenessEvent : public Event { // flush_table() overwrites it from the entry's own tag (track() zeroes it // at insert). int64_t leak_tag = 0; + // LivenessTracker::leakTagIncarnation(leak_tag), read by flush_table() under the exclusive + // _table_lock (so it is the owner's stamp); 0 when untagged. Serialized as + // HeapLiveObject.leakTagIncarnation: leak tags are recycled, so (leakTag, leakTagIncarnation) + // identifies the object across chunks. + u32 leak_tag_incarnation = 0; Context _ctx; }; @@ -139,9 +144,11 @@ class ReferenceChainEvent : public Event { u64 _target_tag; u32 _depth; u8 _root_kind; - // In-memory only, never serialized: LivenessTracker's ownership-period stamp of _target_tag when - // it is a leak tag, captured when the tag was bound to the target's frontier entry. Leak tags are - // recycled, so (_target_tag, _leak_tag_incarnation) identifies the target; 0 otherwise. + // LivenessTracker's ownership-period stamp of _target_tag when it is a leak tag, captured when the + // tag was bound to the target's frontier entry; 0 otherwise. Serialized as + // ReferenceChain.leakTagIncarnation. Leak tags are recycled, so consumers join + // ReferenceChain(targetTag, leakTagIncarnation) to HeapLiveObject(leakTag, leakTagIncarnation): + // the tag alone is only unambiguous within one chunk. u32 _leak_tag_incarnation; std::vector _hops; diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 98455483d8..e5c432603c 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -2423,12 +2423,14 @@ void Recording::recordHeapLiveObject(Buffer *buf, int tid, u64 call_trace_id, ? ((event->_alloc._weight * event->_alloc._size) + event->_skipped) / event->_alloc._size : 0); - // leak_tag precedes the context snapshot to match the metadata field - // order (jfrMetadata.cpp: leakTag is declared between weight and spanId, - // before || contextAttributes) - a field written on the opposite side of - // writeContextSnapshot() than where the metadata declares it makes every - // parser read it from the first context-attribute byte. + // leak_tag and leak_tag_incarnation precede the context snapshot to match + // the metadata field order (jfrMetadata.cpp: leakTag, leakTagIncarnation + // are declared between weight and spanId, before || contextAttributes) - a + // field written on the opposite side of writeContextSnapshot() than where + // the metadata declares it makes every parser read it from the first + // context-attribute byte. buf->putVar64(event->leak_tag); + buf->putVar64((u64)event->leak_tag_incarnation); writeContextSnapshot(buf, event->_ctx); writeEventSizePrefix(buf, start); flushIfNeeded(buf); @@ -2524,6 +2526,7 @@ void Recording::recordReferenceChain(Buffer *buf, ReferenceChainEvent *event) { buf->putVar64(T_REFERENCE_CHAIN); buf->putVar64(event->_start_time); buf->putVar64(event->_target_tag); + buf->putVar64((u64)event->_leak_tag_incarnation); buf->putVar32(event->_depth); buf->putUtf8(root_kind_name); // Original (pre-truncation) chain length, so a consumer can tell a full diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index bee39b415e..f660e85c3b 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -68,7 +68,8 @@ const int MAX_VAR32_LENGTH = 5; // a release-mode write past the buffer). const int REFERENCE_CHAIN_EVENT_FIXED_BYTES = MAX_VAR32_LENGTH /* multi-byte event size prefix */ + - 3 * MAX_VAR64_LENGTH /* type id, start_time, target_tag */ + + 4 * MAX_VAR64_LENGTH /* type id, start_time, target_tag, + leak_tag_incarnation */ + 3 * MAX_VAR32_LENGTH /* depth, totalHops, chain count */ + 32 /* rootKind string, generous */ + MAX_VAR32_LENGTH /* edges count */; diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index 89db648616..a38fe15941 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1109,6 +1109,7 @@ Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) // no-op stubs below. #ifdef DEBUG #include "livenessTracker.h" +#include "objectSampler.h" #include "referenceChains.h" #include @@ -1241,13 +1242,48 @@ Java_com_datadoghq_profiler_JavaProfiler_pollReferenceChainTargets0( ReferenceChainTracker::instance()->finishLoopIterationSerialized(jvmti, env); } +// Counts only the drained chains whose leaf hop (hops are stored leaf-first) is `leaf`'s class, +// so a chain left cached by an earlier test in the same JVM cannot satisfy the caller. The class +// id is resolved with the same GetClassSignature + normalizeClassSignature() + lookupClass() +// sequence the chain collector uses (LivenessTracker::resolveKlassId()). extern "C" DLLEXPORT jint JNICALL -Java_com_datadoghq_profiler_JavaProfiler_drainReferenceChainEventCount0( - JNIEnv *env, jclass unused) { +Java_com_datadoghq_profiler_JavaProfiler_drainReferenceChainEventCountForLeaf0( + JNIEnv *env, jclass unused, jobject leaf) { ProfiledThread::initCurrentThreadSignalSafe(); + jvmtiEnv *jvmti = VM::jvmti(); + if (leaf == nullptr || jvmti == nullptr) { + return 0; + } + u32 leaf_id = 0; + jclass clz = env->GetObjectClass(leaf); + if (clz != nullptr) { + char *sig = nullptr; + if (jvmti->GetClassSignature(clz, &sig, nullptr) == JVMTI_ERROR_NONE && + sig != nullptr) { + const char *name = nullptr; + size_t name_len = 0; + if (ObjectSampler::normalizeClassSignature(sig, &name, &name_len)) { + int id = Profiler::instance()->lookupClass(name, name_len); + if (id > 0) { + leaf_id = (u32)id; + } + } + jvmti->Deallocate((unsigned char *)sig); + } + env->DeleteLocalRef(clz); + } + if (leaf_id == 0) { + return 0; + } std::vector events; ReferenceChainTracker::instance()->drainPendingChainEvents(&events); - return (jint)events.size(); + jint count = 0; + for (const ReferenceChainEvent &event : events) { + if (!event._hops.empty() && event._hops[0].klass_id == leaf_id) { + count++; + } + } + return count; } extern "C" DLLEXPORT void JNICALL @@ -1429,8 +1465,8 @@ Java_com_datadoghq_profiler_JavaProfiler_pollReferenceChainTargets0( JNIEnv *env, jclass unused) {} extern "C" DLLEXPORT jint JNICALL -Java_com_datadoghq_profiler_JavaProfiler_drainReferenceChainEventCount0( - JNIEnv *env, jclass unused) { +Java_com_datadoghq_profiler_JavaProfiler_drainReferenceChainEventCountForLeaf0( + JNIEnv *env, jclass unused, jobject leaf) { return 0; } diff --git a/ddprof-lib/src/main/cpp/jfrMetadata.cpp b/ddprof-lib/src/main/cpp/jfrMetadata.cpp index 51dd36dd92..86b0e8ebe7 100644 --- a/ddprof-lib/src/main/cpp/jfrMetadata.cpp +++ b/ddprof-lib/src/main/cpp/jfrMetadata.cpp @@ -202,6 +202,8 @@ void JfrMetadata::initialize( << field("size", T_LONG, "Original Size", F_BYTES) << field("weight", T_FLOAT, "Sample weight") << field("leakTag", T_LONG, "Leak Tag", F_UNSIGNED) + << field("leakTagIncarnation", T_LONG, + "Leak Tag Incarnation", F_UNSIGNED) << field("spanId", T_LONG, "Span ID") << field("localRootSpanId", T_LONG, "Local Root Span ID") || contextAttributes) @@ -211,6 +213,8 @@ void JfrMetadata::initialize( << category("Datadog", "Profiling") << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) << field("targetTag", T_LONG, "Frontier Tag", F_UNSIGNED) + << field("leakTagIncarnation", T_LONG, "Leak Tag Incarnation", + F_UNSIGNED) << field("depth", T_INT, "Depth") << field("rootKind", T_STRING, "GC Root Kind") << field("totalHops", T_INT, diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp index 484650fec4..538854d9f6 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ b/ddprof-lib/src/main/cpp/livenessTracker.cpp @@ -1929,6 +1929,9 @@ void LivenessTracker::flush_table(std::set *tracked_thread_ids) { event._skipped = _table[i].skipped; event._ctx = _table[i].ctx; event.leak_tag = _table[i].leak_tag; + // The exclusive _table_lock keeps the tag's owner fixed (release only happens in + // cleanup_table()), so this is the current owner's stamp. + event.leak_tag_incarnation = leakTagIncarnation(_table[i].leak_tag); int class_id = 0; if (_table[i].cached_klass_id != 0 && class_map_generation_matches) { diff --git a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp index fb2b641c35..7290ce94a0 100644 --- a/ddprof-lib/src/main/cpp/referenceChainEvents.cpp +++ b/ddprof-lib/src/main/cpp/referenceChainEvents.cpp @@ -447,10 +447,21 @@ void ReferenceChainTracker::pollWatchedTargets(jvmtiEnv *jvmti, JNIEnv *jni) { // Inserts or refreshes klass_id's resolved chain - see _resolved_chains' comment // (referenceChains.h) for why a resolved chain is cached and re-emitted rather than emitted once. +// Returns false when the chain was not stored: the cache is full, or the chain targets a leak tag +// whose owner already died. bool ReferenceChainTracker::cacheResolvedChain(jlong source_tag, ReferenceChainEvent &&event, jlong source_tag_val, u64 source_search_ns) { + if (isLeakTag((jlong)event._target_tag) && !chainTargetsCurrentLeakOwner(event)) { + // Built from a frontier binding whose leak-tag owner already died. Ownership stamps only + // advance, so it can never become valid again; caching it would only take a slot until the + // next dump evicts it. A release racing past this check is evicted by that dump. + TEST_LOG("ReferenceChainTracker::cacheResolvedChain rejected dead-owner chain " + "source_tag=%lld target_tag=%lld", + (long long)source_tag, (long long)event._target_tag); + return false; + } { ExclusiveLockGuard guard(&_resolved_chains_lock); auto it = _resolved_chains.find(source_tag); @@ -816,17 +827,22 @@ void ReferenceChainTracker::drainPendingChainEvents( u64 now = TSC::ticks(); { ExclusiveLockGuard guard(&_resolved_chains_lock); - for (const auto &kv : _resolved_chains) { - const ReferenceChainEvent &event = kv.second.event; + for (auto it = _resolved_chains.begin(); it != _resolved_chains.end();) { + const ReferenceChainEvent &event = it->second.event; if (isLeakTag((jlong)event._target_tag) && !chainTargetsCurrentLeakOwner(event)) { // The leak-tagged target died (its tag was released, and possibly re-acquired by another - // object): emitting targetTag = L would let the consumer's ReferenceChain.targetTag -> - // HeapLiveObject.leakTag join attribute this chain to L's current owner. Skip it, but do - // not erase it - the existing eviction policy decides when it leaves the cache. + // object), so the chain no longer describes the object (targetTag, leakTagIncarnation) + // names. Ownership stamps only advance, so the mismatch is permanent: erase the chain now + // instead of letting it occupy a MAX_RESOLVED_CHAINS slot for the rest of the search. + TEST_LOG("ReferenceChainTracker::drainPendingChainEvents evicted dead-owner chain " + "source_tag=%lld target_tag=%lld", + (long long)it->first, (long long)event._target_tag); + it = _resolved_chains.erase(it); continue; } out->push_back(event); out->back()._start_time = now; + ++it; } } TEST_LOG_SUMMARY("ReferenceChainTracker::drainPendingChainEvents re-emitted=%d", diff --git a/ddprof-lib/src/main/cpp/referenceChains.h b/ddprof-lib/src/main/cpp/referenceChains.h index 23b6cc2632..bfde3616ab 100644 --- a/ddprof-lib/src/main/cpp/referenceChains.h +++ b/ddprof-lib/src/main/cpp/referenceChains.h @@ -998,7 +998,9 @@ class ReferenceChainTracker { // Inserts (or refreshes) klass_id's resolved chain in _resolved_chains, recording the // source_tag/source_search_ns it was reconstructed from so a later poll can tell a stale entry - // from a current one. + // from a current one. Returns false when nothing was stored: the cache is full (counted in + // REFERENCE_CHAIN_EVENTS_DROPPED), or the chain targets a leak tag whose owner already died + // (fails chainTargetsCurrentLeakOwner(); not counted - it is a correctness reject). bool cacheResolvedChain(jlong source_tag, ReferenceChainEvent &&event, jlong source_tag_val, u64 source_search_ns); @@ -1029,8 +1031,9 @@ class ReferenceChainTracker { // True iff `e` targets a leak tag AND was built during that tag's current ownership period: // e._leak_tag_incarnation == LivenessTracker::leakTagIncarnation(e._target_tag). Leak tags are - // recycled once their object dies, while _resolved_chains keeps the dead object's chain, so tag - // equality alone would attribute it to the tag's next owner. False for a non-leak (frontier-tag) + // recycled once their object dies, so tag equality alone would attribute a dead object's chain to + // the tag's next owner; the (tag, stamp) pair is the identity, in memory and in the serialized + // ReferenceChain/HeapLiveObject leakTagIncarnation fields. False for a non-leak (frontier-tag) // target - each caller decides what that means for it. Lock-free (one acquire load); never takes // LivenessTracker's pool lock, so it is safe under _resolved_chains_lock. static bool chainTargetsCurrentLeakOwner(const ReferenceChainEvent &e); @@ -1320,9 +1323,9 @@ class ReferenceChainTracker { // repeatable snapshot, not a drain, so the same live sample's chain is re-emitted into every JFR // chunk it survives into (see _resolved_chains' own comment). Exception: a chain whose target is // a leak tag but which fails chainTargetsCurrentLeakOwner() (its object died and the tag may have - // a new owner) is not emitted - a consumer joins ReferenceChain.targetTag to - // HeapLiveObject.leakTag, which would attribute it to the new owner. Such a chain is not erased - // either; the existing cache eviction policy still decides when it leaves. + // a new owner) is not emitted - it no longer describes the object that + // HeapLiveObject(leakTag, leakTagIncarnation) names. Such a chain is erased from the cache: + // ownership stamps only advance, so it can never become valid again. void drainPendingChainEvents(std::vector *out); // Appends every abandoned-search event queued since the last call and clears the queue - a true diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 615e9d6959..ae90f59124 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -468,7 +468,7 @@ public Map getDebugCounters() { * against. Must be called ON the leaking thread, or the seeded tid will match no * tracked instance and no leak tag will be assigned. */ - public static int getTid() { + static int getTid() { return getTid0(); } @@ -549,7 +549,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * bypassing {@code initialize()}'s live-JVM requirement. Returns {@code true} on * debug builds. */ - public static native boolean setGcGenerationsEnabled0(boolean enabled); + static native boolean setGcGenerationsEnabled0(boolean enabled); /** * Test seam (debug native builds only): seeds one epoch's worth of population @@ -558,7 +558,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * {@code epoch} values) build up a trend {@link #selectLeakCandidateKlassIds0()} * can then rank. */ - public static native void seedKlassPopulationSample0(int klassId, int count, long epoch); + static native void seedKlassPopulationSample0(int klassId, int count, long epoch); /** * Test seam (debug native builds only): seeds one epoch's worth of per-tid trend @@ -569,7 +569,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * must come from {@link #getTid()} whenever the scenario relies on leak tagging of * real tracked instances. */ - public static native void seedTidTrendSample0(int klassId, int tid, int count, long epoch); + static native void seedTidTrendSample0(int klassId, int tid, int count, long epoch); /** * Test seam (debug native builds only): wires {@code representative} in as {@code klassId}'s @@ -579,13 +579,13 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * {@link #pollReferenceChainTargets0()}'s bridging step run without the real sampler or * root-seeded walk. */ - public static native void setKlassPopulationRepresentativeForTest0(int klassId, Object representative); + static native void setKlassPopulationRepresentativeForTest0(int klassId, Object representative); /** * Test seam (debug native builds only): clears LivenessTracker's per-klass population table, * so a later test in the same JVM does not observe leak candidates seeded by an earlier one. */ - public static native void resetKlassPopulationForTest0(); + static native void resetKlassPopulationForTest0(); /** * Test seam (debug native builds only): returns the klass ids LivenessTracker's @@ -593,7 +593,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * selects - the same call ReferenceChainTracker's restart gate and target-polling * bridge use in production. Does not require a reference-chain search to be running. */ - public static native int[] selectLeakCandidateKlassIds0(); + static native int[] selectLeakCandidateKlassIds0(); /** * Test seam (debug native builds only): tags {@code target} and inserts it @@ -604,7 +604,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * {@code datadog.ReferenceChain} event reports), or {@code 0} on failure (reference * chains disabled, or the frontier table is at capacity). */ - public static native long tagAsReferenceChainRoot0(Object target); + static native long tagAsReferenceChainRoot0(Object target); /** * Test seam (debug native builds only): runs exactly one bounded BFS pass of the @@ -612,7 +612,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * background thread/cadence. Returns {@code false} if reference chains are * disabled or the tracker was never started. */ - public static native boolean runReferenceChainPass0(); + static native boolean runReferenceChainPass0(); /** * Test seam (debug native builds only): runs one poll of @@ -621,15 +621,16 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * {@link #runReferenceChainPass0()} walk, reconstructs and queues its chain * event, rather than waiting on the background thread's own scheduling cycle. */ - public static native void pollReferenceChainTargets0(); + static native void pollReferenceChainTargets0(); /** - * Test seam (debug native builds only): drains and returns the number of - * reference-chain events queued by {@link #pollReferenceChainTargets0()} so far - * (the same queue {@code Profiler.dump()} drains in production to write - * {@code datadog.ReferenceChain} JFR events). + * Test seam (debug native builds only): drains the reference-chain events queued by + * {@link #pollReferenceChainTargets0()} so far (the same queue {@code Profiler.dump()} drains + * in production to write {@code datadog.ReferenceChain} JFR events) and returns how many of + * them have {@code leaf}'s class as their leaf (first) hop. Chains for other targets, e.g. left + * cached by an earlier test in the same JVM, are not counted. */ - public static native int drainReferenceChainEventCount0(); + static native int drainReferenceChainEventCountForLeaf0(Object leaf); /** * Test seam (debug native builds only): resets ReferenceChainTracker's search/frontier state @@ -637,7 +638,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * is a process-wide singleton and runPass() walks from the roots only once per search, so an * in-process test needing its own root-seeded walk must call this first. */ - public static native void resetReferenceChainSearchForTest0(); + static native void resetReferenceChainSearchForTest0(); /** * Test seam (debug native builds only): diagnostic-only, does not tag {@code target}. Reads @@ -646,14 +647,14 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * (0 = expands next) if still queued, {@code -1} if tagged but no longer queued (already * expanded), or {@code -2} if never admitted at all. */ - public static native long getReferenceChainPendingPositionForTest0(Object target); + static native long getReferenceChainPendingPositionForTest0(Object target); /** * Test seam (debug native builds only): the current size of ReferenceChainTracker's * pending-expansion queue, for computing {@link #getReferenceChainPendingPositionForTest0}'s * position as a fraction of the current backlog. */ - public static native long getReferenceChainPendingSizeForTest0(); + static native long getReferenceChainPendingSizeForTest0(); /** * Test seam (-DDEBUG native builds only - debug/asan/tsan; returns -1 in release): does not @@ -664,7 +665,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * {@code -1} if target has no frontier entry (never admitted, or still carrying only a leak * tag). */ - public static native int getReferenceChainFrontierStateForTest0(Object target); + static native int getReferenceChainFrontierStateForTest0(Object target); /** * Test seam (-DDEBUG native builds only - debug/asan/tsan; returns 0 in release): does not @@ -675,7 +676,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * ownership of that leak tag: leak tags are recycled after their object dies, so identity is * the (leak tag, ownership-period stamp) pair, not the tag alone. */ - public static native long getReferenceChainResolvedTargetTagForTest0(Object target); + static native long getReferenceChainResolvedTargetTagForTest0(Object target); /** * Test seam (debug native builds only): seeds one heap-floor-ring sample - the input to @@ -684,14 +685,14 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * compared against each other, never against a real wall clock, so any strictly increasing * sequence is valid. */ - public static native void heapFloorRecordForTest0(long usedBytes, long timestampNs); + static native void heapFloorRecordForTest0(long usedBytes, long timestampNs); /** * Test seam (debug native builds only): overrides the max-heap-size {@code secondsToOOM()} * projects against, bypassing the real {@code Runtime.maxMemory()} resolution, so the * projection does not depend on the test JVM's {@code -Xmx}. */ - public static native void setMaxHeapBytesForTest0(long maxHeapBytes); + static native void setMaxHeapBytesForTest0(long maxHeapBytes); /** * Test seam (debug native builds only): temporarily disables {@code onGC()}'s own @@ -700,17 +701,19 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * with a real {@code OS::nanotime()} timestamp and real heap usage, corrupting * {@code secondsToOOM()}'s projection. Pass {@code false} to disable, {@code true} to restore. */ - public static native void setHeapFloorRecordingForTest0(boolean enabled); + static native void setHeapFloorRecordingForTest0(boolean enabled); /** * Test seam (debug native builds only): reports whether ReferenceChainTracker's search-restart * gate ({@code canAffordNewSearch()} -> {@code hasLeakSignal()}) would currently allow a * fresh/terminal search to start - in particular, whether {@code secondsToOOM()}'s urgent-OOM * bypass opens this gate even with zero per-klass leak candidates. Unlike - * {@link #runReferenceChainPass0()}, which calls {@code runPass()} unconditionally, this reads - * the gate without running a pass. + * {@link #runReferenceChainPass0()}, which calls {@code runPass()} unconditionally, this + * evaluates the gate without running a pass. This is not a read-only query: under the native + * engine lock it may account a completed search's pain and restart/reset terminal search + * state. */ - public static native boolean shouldRunPassForTest0(); + static native boolean shouldRunPassForTest0(); /** * Test seam (debug native builds only): the number of BFS passes run for the current/most @@ -719,7 +722,7 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * therefore {@code collectStaleExpandedEntriesForRotation()}) ran strictly after the object * existed, not from a pass racing its creation. */ - public static native int referenceChainPassesRunForTest0(); + static native int referenceChainPassesRunForTest0(); // ---- Test-only reads of the current thread's OTEP record ---------------------------------- // Each resolves the current carrier's record directly (like the write primitives above) with diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp index c0b0630dff..0ec21c4549 100644 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp @@ -2682,6 +2682,70 @@ TEST_F(LivenessTrackerMockJvmTest, SharedObjectLeakTagHasSingleOwner) { << "the object's leak tag must go back to the pool exactly once"; } +// start() rebuilds the leak-tag pool from the surviving tracking table. A tag +// whose owner is unchanged (a table row still holds it) must keep its stamp, so +// chains cached for that object stay attributable; a tag whose in-use bit flips +// (held without a row owning it) starts a new ownership period; a tag that +// stays free keeps its stamp. +TEST_F(LivenessTrackerMockJvmTest, StartRebuildBumpsStampOnlyWhenOwnershipFlips) { + LivenessTracker *tracker = LivenessTracker::instance(); + u32 retained_id = classIdOf(kRetainedSignature); + ASSERT_NE(0u, retained_id); + + constexpr jint kTid = 7; + jobject obj = trackNew(&jvm.retained_class, kTid); + jobject reclaimed_obj = trackNew(&jvm.retained_class, kTid); + // Resolves the rows' cached class id, which tagLeakInstances() matches on. + sweep(); + KlassCandidate candidate{}; + candidate.klass_id = retained_id; + candidate.qualifying_tids[0] = kTid; + candidate.qualifying_tid_count = 1; + ASSERT_EQ(2, tracker->tagLeakInstances(&jvm.jvmti, &candidate, 1)); + jlong row_tag = 0; + mockGetTag(&jvm.jvmti, obj, &row_tag); + ASSERT_GE(row_tag, tracker->leakTagBaseForTest()); + + // In use, but no table row owns it: the rebuild frees it. + jlong orphan_tag = tracker->acquireLeakTagForTest(); + ASSERT_GE(orphan_tag, tracker->leakTagBaseForTest()); + // Free before and after the rebuild. + jlong free_tag = tracker->acquireLeakTagForTest(); + ASSERT_GE(free_tag, tracker->leakTagBaseForTest()); + tracker->releaseLeakTagForTest(free_tag); + + // A row still holds it, but the pool lists it free: the rebuild reclaims it. + // Released last, so no acquire above pops it back off the LIFO free list. + jlong reclaimed_tag = 0; + mockGetTag(&jvm.jvmti, reclaimed_obj, &reclaimed_tag); + ASSERT_GE(reclaimed_tag, tracker->leakTagBaseForTest()); + ASSERT_NE(row_tag, reclaimed_tag); + tracker->releaseLeakTagForTest(reclaimed_tag); + ASSERT_FALSE(tracker->leakTagInUseForTest(reclaimed_tag)); + + u32 row_stamp = tracker->leakTagIncarnation(row_tag); + u32 reclaimed_stamp = tracker->leakTagIncarnation(reclaimed_tag); + u32 orphan_stamp = tracker->leakTagIncarnation(orphan_tag); + u32 free_stamp = tracker->leakTagIncarnation(free_tag); + + Arguments args; + ASSERT_FALSE(args.parse("generations=true")); + ASSERT_FALSE(tracker->start(args)); + + EXPECT_TRUE(tracker->leakTagInUseForTest(row_tag)); + EXPECT_EQ(row_stamp, tracker->leakTagIncarnation(row_tag)) + << "a tag whose owning row survived the rebuild must keep its stamp"; + EXPECT_TRUE(tracker->leakTagInUseForTest(reclaimed_tag)); + EXPECT_NE(reclaimed_stamp, tracker->leakTagIncarnation(reclaimed_tag)) + << "reclaiming a free-listed tag for its row is an ownership transition"; + EXPECT_FALSE(tracker->leakTagInUseForTest(orphan_tag)); + EXPECT_NE(orphan_stamp, tracker->leakTagIncarnation(orphan_tag)) + << "freeing an in-use tag in the rebuild is an ownership transition"; + EXPECT_FALSE(tracker->leakTagInUseForTest(free_tag)); + EXPECT_EQ(free_stamp, tracker->leakTagIncarnation(free_tag)) + << "a tag that stays free keeps its stamp"; +} + // Profiler::start() resets the class map, restarting its ids at 1, while the // tracking table and its cached class ids survive. A flush before the next GC // must resolve the class again instead of emitting the cached id, which now diff --git a/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp b/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp index 9ff0562a71..74ff19d803 100644 --- a/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp +++ b/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp @@ -234,12 +234,15 @@ class ReferenceChainJfrRoundtripTest : public ::testing::Test { }; // Path agreed with the companion Java parser test in ddprof-test, which -// reads the same file back via JMC's -// JfrLoaderToolkit. Both sides resolve it via the OS temp dir so the -// producer (this gtest) and the consumer (the Java test, run afterwards by -// the same operator/CI job on the same machine) agree without either side -// needing to know the other module's build directory layout. +// reads the same file back via JMC's JfrLoaderToolkit. The build passes the +// same DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR to both sides (ddprof-lib and +// ddprof-test build.gradle.kts) and deletes the file before this gtest runs; +// the OS temp dir is only a fallback for running the binary by hand. static std::string chainRoundtripJfrPath() { + const char *configured = getenv("DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR"); + if (configured != nullptr && *configured != 0) { + return configured; + } const char *tmp = getenv("TMPDIR"); std::string dir = (tmp != nullptr && *tmp != 0) ? tmp : "/tmp"; if (dir.back() != '/') { @@ -345,6 +348,8 @@ TEST_F(ReferenceChainJfrRoundtripTest, ProducesValidStandaloneJfrWithChainEvent) // pins the JFR encoding, so the event is built directly). ReferenceChainEvent event; event._target_tag = 3; + // Non-zero so the Java parser test can pin the serialized field. + event._leak_tag_incarnation = 5; event._depth = 2; event._root_kind = 21; // JVMTI_HEAP_REFERENCE_JNI_GLOBAL - exercises the // rootKind field end to end (the engine's admission @@ -481,6 +486,7 @@ TEST_F(ReferenceChainJfrRoundtripTest, TruncatesOversizeChainAndKeepsEmittedLabe off += sizeOfVar64((u64)T_REFERENCE_CHAIN); // type id off += sizeOfVar64(event._start_time); off += sizeOfVar64(event._target_tag); + off += sizeOfVar64((u64)event._leak_tag_incarnation); off += sizeOfVar64(event._depth); // depth, small by construction // rootKind putUtf8: 1 encoding-tag byte + var32 length + payload. const char *root_kind_name = "jni_global"; // rootKindName(21) @@ -511,4 +517,5 @@ TEST_F(ReferenceChainJfrRoundtripTest, TruncatesOversizeChainAndKeepsEmittedLabe TEST_F(ReferenceChainJfrRoundtripTest, DefaultLivenessEventHasZeroLeakTag) { ObjectLivenessEvent event; EXPECT_EQ(0, event.leak_tag); + EXPECT_EQ(0u, event.leak_tag_incarnation); } diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc index d580e57aaa..096af1e805 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -862,7 +862,7 @@ TEST_F(ResolvedChainCacheTest, LeakTagChainOfCurrentOwnerIsResolvedAndEmitted) { // ABA: the chain's object dies (tag released), and the tag is recycled for a new // object. The old chain must not be attributed to the new owner by the seam nor emitted into the -// dump - and it must not be erased either (the eviction policy owns removal). +// dump, and the dump evicts it (ownership stamps only advance, so it can never become valid). TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitted) { LivenessTracker *lt = LivenessTracker::instance(); jlong leak = lt->acquireLeakTagForTest(); @@ -876,8 +876,8 @@ TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitte lt->releaseLeakTagForTest(leak); EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(leak)); EXPECT_FALSE(drainContainsTarget((u64)leak)); - EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()) - << "emission must not erase the stale chain"; + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()) + << "the dump must evict the dead owner's chain"; // A new object receives the recycled tag (LIFO). jlong reused = lt->acquireLeakTagForTest(); @@ -885,7 +885,7 @@ TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitte EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(reused)) << "the dead owner's chain was attributed to the tag's new owner"; EXPECT_FALSE(drainContainsTarget((u64)reused)); - EXPECT_EQ(1u, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); // Once the new owner's own chain is cached, it is the one that matches. ReferenceChainsTestAccessor::cacheChain( @@ -897,6 +897,20 @@ TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitte EXPECT_EQ(lt->leakTagIncarnation(reused), out[0]._leak_tag_incarnation); } +// A chain built from a binding whose leak-tag owner already died is refused by the cache: it +// would only occupy a MAX_RESOLVED_CHAINS slot until the next dump evicts it. +TEST_F(ResolvedChainCacheTest, DeadOwnerChainIsNotCached) { + LivenessTracker *lt = LivenessTracker::instance(); + jlong leak = lt->acquireLeakTagForTest(); + ASSERT_TRUE(ReferenceChainTracker::isLeakTag(leak)); + u32 stale_stamp = lt->leakTagIncarnation(leak); + lt->releaseLeakTagForTest(leak); + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ReferenceChainsTestAccessor::cacheChain(5, makeLeakEvent(leak, stale_stamp), 5, 1000); + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); + EXPECT_FALSE(drainContainsTarget((u64)leak)); +} + // Within one ownership period the leak-tag path is generation-agnostic: a chain an earlier search // cached for the same owner still answers after a restart. TEST_F(ResolvedChainCacheTest, LeakTagChainMatchesAcrossGenerationsForSameOwner) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java index b62579f2e1..e0f79c7046 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package com.datadoghq.profiler; import java.io.BufferedReader; @@ -36,6 +41,15 @@ public enum LineConsumerResult { IGNORE } + /** + * Scales a child-process wait budget for sanitizer builds (asan/tsan), whose native code runs + * several times slower than debug/release. + */ + protected static long childTimeoutSeconds(long baseSeconds) { + String config = System.getProperty("ddprof_test.config"); + return config != null && config.endsWith("san") ? baseSeconds * 3 : baseSeconds; + } + protected final LaunchResult launch(String target, List jvmArgs, String commands, Function onStdoutLine, Function onStderrLine) throws Exception { return launch(target, jvmArgs, commands, Collections.emptyMap(), onStdoutLine, onStderrLine); } @@ -73,6 +87,17 @@ protected final LaunchResult launch(String target, List jvmArgs, String ProcessBuilder pb = new ProcessBuilder(args); pb.environment().putAll(env); Process p = pb.start(); + try { + return awaitChild(p, timeoutSeconds, onStdoutLine, onStderrLine); + } finally { + // Covers an interrupted launch() as well as a child that outlived destroyForcibly(). + if (p.isAlive()) { + p.destroyForcibly(); + } + } + } + + private static LaunchResult awaitChild(Process p, long timeoutSeconds, Function onStdoutLine, Function onStderrLine) throws Exception { Thread stdoutReader = new Thread(() -> { Function lineProcessor = onStdoutLine != null ? onStdoutLine : l -> LineConsumerResult.CONTINUE; try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()))) { @@ -156,6 +181,8 @@ protected final LaunchResult launch(String target, List jvmArgs, String // and a reader still alive after the bound is left behind. stdoutReader.join(READER_JOIN_TIMEOUT_MS); stderrReader.join(READER_JOIN_TIMEOUT_MS); - return new LaunchResult(val, p.exitValue()); + // exitValue() throws while the child is still alive (it survived destroyForcibly()); report + // -1 instead so the caller still sees inTime=false rather than an IllegalThreadStateException. + return new LaunchResult(val, p.isAlive() ? -1 : p.exitValue()); } -} \ No newline at end of file +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java index 313f3ca19d..6681ddc1d8 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java @@ -139,6 +139,20 @@ private static void entryFrameBurn(long millis) { entryFrameSink = acc; } + /** + * Splits a scenario mode's single {@code "|||"} argument + * (packed so launch()'s (target, commands) contract needs no extra argument). + */ + private static String[] unpackScenarioArgs(String mode, String[] args) { + String packed = args.length == 2 ? args[1] : ""; + int sep = packed.indexOf("|||"); + if (sep < 0) { + throw new IllegalArgumentException( + mode + " requires \"|||\", got: " + packed); + } + return new String[] {packed.substring(0, sep), packed.substring(sep + "|||".length())}; + } + public static void main(String[] args) throws Exception { Thread worker = null; try { @@ -222,16 +236,9 @@ public static void main(String[] args) throws Exception { } } } else if (args[0].equals("leak-cache")) { - // "|||" packed into args[1] so launch()'s - // (target, commands) contract needs no extra argument. - String packed = args.length == 2 ? args[1] : ""; - int sep = packed.indexOf("|||"); - if (sep < 0) { - throw new IllegalArgumentException( - "leak-cache requires \"|||\", got: " + packed); - } - String commands = packed.substring(0, sep); - String scratchPath = packed.substring(sep + "|||".length()); + String[] packed = unpackScenarioArgs("leak-cache", args); + String commands = packed[0]; + String scratchPath = packed[1]; JavaProfiler instance = JavaProfiler.getInstance(); // run() starts the profiler itself: it must seed its cache fixture first. LeakingCacheScenario.run(instance, commands, Paths.get(scratchPath)); @@ -240,43 +247,28 @@ public static void main(String[] args) throws Exception { // result line has already been printed. System.exit(0); } else if (args[0].equals("leak-static-field")) { - // Same "|||" packing as leak-cache above. - String packed = args.length == 2 ? args[1] : ""; - int sep = packed.indexOf("|||"); - if (sep < 0) { - throw new IllegalArgumentException( - "leak-static-field requires \"|||\", got: " + packed); - } - String commands = packed.substring(0, sep); - String scratchPath = packed.substring(sep + "|||".length()); + // Same packing as leak-cache (unpackScenarioArgs()). + String[] packed = unpackScenarioArgs("leak-static-field", args); + String commands = packed[0]; + String scratchPath = packed[1]; JavaProfiler instance = JavaProfiler.getInstance(); StaticFieldGrowingCollectionScenario.run(instance, commands, Paths.get(scratchPath)); System.out.flush(); System.exit(0); } else if (args[0].equals("leak-correlation")) { - // Same "|||" packing as leak-cache above. - String packed = args.length == 2 ? args[1] : ""; - int sep = packed.indexOf("|||"); - if (sep < 0) { - throw new IllegalArgumentException( - "leak-correlation requires \"|||\", got: " + packed); - } - String commands = packed.substring(0, sep); - String scratchPath = packed.substring(sep + "|||".length()); + // Same packing as leak-cache (unpackScenarioArgs()). + String[] packed = unpackScenarioArgs("leak-correlation", args); + String commands = packed[0]; + String scratchPath = packed[1]; JavaProfiler instance = JavaProfiler.getInstance(); LeakTagCorrelationScenario.run(instance, commands, Paths.get(scratchPath)); System.out.flush(); System.exit(0); } else if (args[0].equals("threadlocal-leak")) { - // Same "|||" packing as leak-cache above. - String packed = args.length == 2 ? args[1] : ""; - int sep = packed.indexOf("|||"); - if (sep < 0) { - throw new IllegalArgumentException( - "threadlocal-leak requires \"|||\", got: " + packed); - } - String commands = packed.substring(0, sep); - String scratchPath = packed.substring(sep + "|||".length()); + // Same packing as leak-cache (unpackScenarioArgs()). + String[] packed = unpackScenarioArgs("threadlocal-leak", args); + String commands = packed[0]; + String scratchPath = packed[1]; JavaProfiler instance = JavaProfiler.getInstance(); ThreadLocalLeakScenario.run(instance, commands, Paths.get(scratchPath)); System.out.flush(); @@ -285,14 +277,9 @@ public static void main(String[] args) throws Exception { // Same packing as threadlocal-leak, but the leak thread is // started only AFTER the recording began, exercising // Profiler::onThreadStart's registerThreadObject() wiring. - String packed = args.length == 2 ? args[1] : ""; - int sep = packed.indexOf("|||"); - if (sep < 0) { - throw new IllegalArgumentException( - "threadlocal-leak-late requires \"|||\", got: " + packed); - } - String commands = packed.substring(0, sep); - String scratchPath = packed.substring(sep + "|||".length()); + String[] packed = unpackScenarioArgs("threadlocal-leak-late", args); + String commands = packed[0]; + String scratchPath = packed[1]; JavaProfiler instance = JavaProfiler.getInstance(); ThreadLocalLeakScenario.run(instance, commands, Paths.get(scratchPath), true); System.out.flush(); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java index b3052f3a20..55766f1890 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java @@ -9,6 +9,9 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -22,6 +25,22 @@ public void ownedBlockHooksAreNotPublicApiBeforeTaskBlockInstrumentation() throw assertNotPublic(JavaProfiler.class.getDeclaredMethod("blockExit", long.class)); } + @Test + public void referenceChainTestSeamsAreNotPublicApi() { + Set seams = new HashSet<>(Arrays.asList( + "getTid", "setGcGenerationsEnabled0", "seedKlassPopulationSample0", + "seedTidTrendSample0", "selectLeakCandidateKlassIds0", "tagAsReferenceChainRoot0", + "runReferenceChainPass0", "pollReferenceChainTargets0", + "drainReferenceChainEventCountForLeaf0")); + for (Method m : JavaProfiler.class.getDeclaredMethods()) { + if (seams.contains(m.getName()) || m.getName().endsWith("ForTest0")) { + assertFalse(Modifier.isPublic(m.getModifiers()), + m.getName() + " is a test seam and must not be public API" + + " (tests reach it through ReferenceChainTestHooks)"); + } + } + } + private static void assertNotPublic(Method method) { assertFalse(Modifier.isPublic(method.getModifiers()), method.getName() + " must remain non-public until PR2 wires TaskBlock instrumentation"); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java new file mode 100644 index 0000000000..a070e32462 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java @@ -0,0 +1,105 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler; + +/** + * Test bridge for the package-private reference-chain/liveness test seams on {@link JavaProfiler}, + * so tests outside {@code com.datadoghq.profiler} can call them without the seams becoming public + * API of the shipped jar. The native seams exist only in -DDEBUG builds (debug/asan/tsan); release + * builds get no-op stubs. + */ +public final class ReferenceChainTestHooks { + private ReferenceChainTestHooks() {} + + /** The calling thread's profiler tid. Call it on the allocating (leaking) thread. */ + public static int getTid() { + return JavaProfiler.getTid(); + } + + public static boolean setGcGenerationsEnabled0(boolean enabled) { + return JavaProfiler.setGcGenerationsEnabled0(enabled); + } + + public static void seedKlassPopulationSample0(int klassId, int count, long epoch) { + JavaProfiler.seedKlassPopulationSample0(klassId, count, epoch); + } + + public static void seedTidTrendSample0(int klassId, int tid, int count, long epoch) { + JavaProfiler.seedTidTrendSample0(klassId, tid, count, epoch); + } + + public static void setKlassPopulationRepresentativeForTest0(int klassId, Object representative) { + JavaProfiler.setKlassPopulationRepresentativeForTest0(klassId, representative); + } + + public static void resetKlassPopulationForTest0() { + JavaProfiler.resetKlassPopulationForTest0(); + } + + public static int[] selectLeakCandidateKlassIds0() { + return JavaProfiler.selectLeakCandidateKlassIds0(); + } + + public static long tagAsReferenceChainRoot0(Object target) { + return JavaProfiler.tagAsReferenceChainRoot0(target); + } + + public static boolean runReferenceChainPass0() { + return JavaProfiler.runReferenceChainPass0(); + } + + public static void pollReferenceChainTargets0() { + JavaProfiler.pollReferenceChainTargets0(); + } + + public static int drainReferenceChainEventCountForLeaf0(Object leaf) { + return JavaProfiler.drainReferenceChainEventCountForLeaf0(leaf); + } + + public static void resetReferenceChainSearchForTest0() { + JavaProfiler.resetReferenceChainSearchForTest0(); + } + + public static long getReferenceChainPendingPositionForTest0(Object target) { + return JavaProfiler.getReferenceChainPendingPositionForTest0(target); + } + + public static long getReferenceChainPendingSizeForTest0() { + return JavaProfiler.getReferenceChainPendingSizeForTest0(); + } + + public static int getReferenceChainFrontierStateForTest0(Object target) { + return JavaProfiler.getReferenceChainFrontierStateForTest0(target); + } + + public static long getReferenceChainResolvedTargetTagForTest0(Object target) { + return JavaProfiler.getReferenceChainResolvedTargetTagForTest0(target); + } + + public static void heapFloorRecordForTest0(long usedBytes, long timestampNs) { + JavaProfiler.heapFloorRecordForTest0(usedBytes, timestampNs); + } + + public static void setMaxHeapBytesForTest0(long maxHeapBytes) { + JavaProfiler.setMaxHeapBytesForTest0(maxHeapBytes); + } + + public static void setHeapFloorRecordingForTest0(boolean enabled) { + JavaProfiler.setHeapFloorRecordingForTest0(enabled); + } + + /** + * Not a read-only query: under the native engine lock the gate may account a completed + * search's pain and restart/reset terminal search state. + */ + public static boolean shouldRunPassForTest0() { + return JavaProfiler.shouldRunPassForTest0(); + } + + public static int referenceChainPassesRunForTest0() { + return JavaProfiler.referenceChainPassesRunForTest0(); + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java index bcc7f9c809..7552854696 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java @@ -6,7 +6,7 @@ package com.datadoghq.profiler.referencechains; import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import com.datadoghq.profiler.Platform; import org.junit.jupiter.api.Test; @@ -56,9 +56,9 @@ protected boolean isPlatformSupported() { } private static void assumeDebugBuild() { - assumeTrue("debug".equals(System.getProperty("ddprof_test.config")), + assumeTrue(ReferenceChainAssertions.isDebugInstrumentedBuild(), "heapFloorRecordForTest0/setMaxHeapBytesForTest0/shouldRunPassForTest0 only exist in a " - + "debug native build (javaApi.cpp's #ifdef DEBUG guard)"); + + "-DDEBUG native build - debug/asan/tsan (javaApi.cpp's #ifdef DEBUG guard)"); } /** @@ -82,24 +82,24 @@ private static void assumeDebugBuild() { public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() throws InterruptedException { assumeDebugBuild(); - JavaProfiler.setHeapFloorRecordingForTest0(false); - JavaProfiler.resetKlassPopulationForTest0(); - JavaProfiler.resetReferenceChainSearchForTest0(); + ReferenceChainTestHooks.setHeapFloorRecordingForTest0(false); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); try { // Baseline BEFORE publishing any urgency input: the live BFS thread may run the one // urgency-authorized search as soon as the samples below land, and that pass must count. - int passesBefore = JavaProfiler.referenceChainPassesRunForTest0(); + int passesBefore = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); // 1000 MiB fake max heap, rising from 100 MiB to 991 MiB over 9 (fake) seconds: ~99 MiB/s, // 9 MiB of headroom left at the last sample -> projected time-to-OOM ~= 9/99 s, far under // OOM_URGENT_THRESHOLD_S = 300s. - JavaProfiler.setMaxHeapBytesForTest0(1000L * 1024 * 1024); + ReferenceChainTestHooks.setMaxHeapBytesForTest0(1000L * 1024 * 1024); for (int i = 0; i < 10; i++) { long usedBytes = (100L + i * 99L) * 1024 * 1024; long timestampNs = i * 1_000_000_000L; - JavaProfiler.heapFloorRecordForTest0(usedBytes, timestampNs); + ReferenceChainTestHooks.heapFloorRecordForTest0(usedBytes, timestampNs); } - int[] candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + int[] candidates = ReferenceChainTestHooks.selectLeakCandidateKlassIds0(); assertTrue(candidates == null || candidates.length == 0, "This test's own precondition: no per-klass candidate should exist, so passes running " + "below can only come from the aggregate urgent-OOM bypass"); @@ -107,21 +107,21 @@ public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() // Idle BFS cadence is ~1s/pass; urgency ramps it to ~10ms after the latch. 20s covers the // first evaluation landing up to one idle cadence after the seeding loop. long deadline = System.currentTimeMillis() + 20_000; - while (JavaProfiler.referenceChainPassesRunForTest0() == passesBefore + while (ReferenceChainTestHooks.referenceChainPassesRunForTest0() == passesBefore && System.currentTimeMillis() < deadline) { Thread.sleep(50); } - assertTrue(JavaProfiler.referenceChainPassesRunForTest0() > passesBefore, + assertTrue(ReferenceChainTestHooks.referenceChainPassesRunForTest0() > passesBefore, "Expected the BFS thread to run passes from the urgent heap-wide OOM projection alone, " + "with zero per-klass leak candidate"); - candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + candidates = ReferenceChainTestHooks.selectLeakCandidateKlassIds0(); assertTrue(candidates == null || candidates.length == 0, "The passes that just ran must still have had zero per-klass candidates - the " + "aggregate urgent-OOM projection was the only possible trigger"); } finally { - JavaProfiler.setMaxHeapBytesForTest0(-1); - JavaProfiler.setHeapFloorRecordingForTest0(true); + ReferenceChainTestHooks.setMaxHeapBytesForTest0(-1); + ReferenceChainTestHooks.setHeapFloorRecordingForTest0(true); } } @@ -134,25 +134,25 @@ public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() @Test public void shouldNotOpenSearchGateOnFlatHeapFloorWithNoLeakCandidate() { assumeDebugBuild(); - JavaProfiler.setHeapFloorRecordingForTest0(false); - JavaProfiler.resetKlassPopulationForTest0(); - JavaProfiler.resetReferenceChainSearchForTest0(); + ReferenceChainTestHooks.setHeapFloorRecordingForTest0(false); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); try { - JavaProfiler.setMaxHeapBytesForTest0(1000L * 1024 * 1024); + ReferenceChainTestHooks.setMaxHeapBytesForTest0(1000L * 1024 * 1024); for (int i = 0; i < 10; i++) { - JavaProfiler.heapFloorRecordForTest0(200L * 1024 * 1024, i * 1_000_000_000L); + ReferenceChainTestHooks.heapFloorRecordForTest0(200L * 1024 * 1024, i * 1_000_000_000L); } - int[] candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + int[] candidates = ReferenceChainTestHooks.selectLeakCandidateKlassIds0(); assertTrue(candidates == null || candidates.length == 0, "This test's own precondition: no per-klass candidate should exist either"); - assertFalse(JavaProfiler.shouldRunPassForTest0(), + assertFalse(ReferenceChainTestHooks.shouldRunPassForTest0(), "Expected the search-restart gate to stay closed: no heap growth and no per-klass " + "candidate give hasLeakSignal() nothing to trust"); } finally { - JavaProfiler.setMaxHeapBytesForTest0(-1); - JavaProfiler.setHeapFloorRecordingForTest0(true); + ReferenceChainTestHooks.setMaxHeapBytesForTest0(-1); + ReferenceChainTestHooks.setHeapFloorRecordingForTest0(true); } } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java index c30bf06472..883594abf0 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -83,7 +83,7 @@ void shouldReconstructReferrerChainInSeparateProcess() throws Exception { Collections.emptyMap(), // Generous: a separate JVM's startup/classloading cost on top of the up-to-25-round // growth loop plus grace period (~20s in-process). - 90, + childTimeoutSeconds(90), line -> { if (line.startsWith(LeakingCacheScenario.FOUND_MARKER) || line.equals(LeakingCacheScenario.NOT_FOUND_MARKER) @@ -163,7 +163,7 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce // own tail wait (StaticFieldGrowingCollectionScenario.run()) needs up to ~60s on top of // the round loop for a cold JVM's first full pass to actually complete - see that // method's own comment. - 150, + childTimeoutSeconds(150), line -> { if (line.startsWith(StaticFieldGrowingCollectionScenario.FOUND_MARKER) || line.equals(StaticFieldGrowingCollectionScenario.NOT_FOUND_MARKER)) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java index 280ba3a149..b0455e247e 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java @@ -21,6 +21,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_BASE; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_POOL_SIZE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -37,39 +39,31 @@ * AND matches a {@code datadog.HeapLiveObject}'s {@code leakTag}. See * {@link LeakTagCorrelationScenario}'s own header for what each ingredient reproduces. * - *

    Diagnostics on failure: unlike the pod-debugging loop this scenario exists to - * replace, the child's {@code TEST_LOG} stream (queue depths, batch sizes, interception - * counts - the same evidence collected off a live pod's logs) arrives on this process's - * stdout consumer. A failing run embeds a filtered tail of it in the assertion message, so - * the failure mode is diagnosable without redeploying anything. + *

    Diagnostics on failure: the child's {@code TEST_LOG} stream (queue depths, batch + * sizes, interception counts) arrives on this process's stdout consumer. A failing run embeds + * a filtered tail of it in the assertion message, so the failure mode is diagnosable from the + * test report alone. */ @Tag("slow") public class LeakTagCorrelationReferenceChainTest extends AbstractProcessProfilerTest { - // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE) - the contract under - // test, asserted against the runtime event values the child reports. - private static final long LEAK_TAG_BASE = 0x40000000L; - private static final long LEAK_TAG_POOL_SIZE = 256; - @Test void shouldCorrelateLeakChainTargetTagWithLiveObjectLeakTag() throws Exception { assertCorrelationFound("pausetarget=60000"); } /** - * The pod-regime variant of the correlation use case: the same scenario under a - * pod-like tight pass window (pausetarget in single-digit ms, as on the production-like - * pod whose expand window was ~10ms against a 14-41ms GetObjectsWithTags floor at a - * 242k-entry tag map). Full correlation cannot be the pass - * criterion at this window width: the scenario's trend gates were tuned for the loose - * window's crawl/GC interleaving, and under tight windows the leak klass's real-fold - * trend can flip within a single run - the same knife-edge the pod itself lives on - * (chains arrive via the plain crawl path while pool tagging/interception loses the - * race; observed: [correlation-tag-out-of-pool] with zero interceptions). What CAN be - * asserted deterministically here is the machinery this window exists to exercise: - * the leak-accumulation rotation must engage and select FRONTIER-state (un-expanded - * backlog) holders - without that, the tier selected zero on the pod while holding 51k - * known parent candidates. GetObjectsWithTags batch control is + * The tight-window variant of the correlation use case: the same scenario under a tight + * pass window (pausetarget in single-digit ms, narrower than a large tag map's + * GetObjectsWithTags cost floor). Full correlation cannot be the pass criterion at this + * window width: the scenario's trend gates are tuned for the loose window's crawl/GC + * interleaving, and under tight windows the leak klass's real-fold trend can flip within a + * single run (chains arrive via the plain crawl path while pool tagging/interception loses + * the race, reported as [correlation-tag-out-of-pool]). What this variant asserts + * deterministically is the machinery the window exists to exercise: the leak-accumulation + * rotation must engage and select FRONTIER-state (un-expanded backlog) holders. Without + * that, the rotation tier selects nothing while known parent candidates pile up in the + * backlog. GetObjectsWithTags batch control is * covered unit-level (gotwWindowNs() widening): a real child JVM's local tag map * (~100k filler) keeps the per-call floor under the nominal window, so the widening * regime cannot be reproduced end-to-end locally without a 242k-entry tag map. @@ -87,9 +81,9 @@ private void assertCorrelationFound(String pauseTargetOption) throws Exception { * Drives the leak-correlation child JVM and asserts the correlated chain, with the * referencechains pausetarget given by {@code pauseTargetOption} (everything else held * fixed across the two callers so the variant differs in exactly the pass-window regime). - * With {@code requireFrontierRotation} (the tight-window pod-regime variant), the + * With {@code requireFrontierRotation} (the tight-window variant), the * leak-accumulation rotation's FRONTIER-state selection is additionally asserted, and - * the correlation result markers that the pod regime legitimately produces without the + * the correlation result markers that the tight window legitimately produces without the * full correlation completing (not-found, tag-out-of-pool - see the tight variant's own * comment) are accepted instead of failed. This variant runs only on DEBUG-instrumented * native builds and forces reference-chains debug level 2 in the child; the loose variant @@ -104,8 +98,7 @@ private void assertCorrelationFound(String pauseTargetOption, // The FRONTIER-rotation signal is a level-2 TEST_LOG line (referenceChainWalk.cpp, // collectLeakAccumulationCandidatesForRotation) - compiled out without -DDEBUG // (debug/asan/tsan only) and silent unless DD_PROFILING_REFERENCE_CHAINS_DEBUG=2. - String config = System.getProperty("ddprof_test.config"); - assumeTrue(config != null && ("debug".equals(config) || config.endsWith("san")), + assumeTrue(ReferenceChainAssertions.isDebugInstrumentedBuild(), "FRONTIER-rotation assertion needs the reference-chains TEST_LOG stream, only " + "compiled into DEBUG-instrumented builds (debug/asan/tsan)"); } @@ -146,15 +139,18 @@ private void assertCorrelationFound(String pauseTargetOption, childEnv, // The scenario's own round loop (up to 25 rounds plus a 5-attempt grace period, // ~40-60s observed) on top of a cold external JVM's startup/classloading cost and - // this scenario's larger graph - same margin logic as the existing scenarios. - 150, + // this scenario's larger graph - same margin logic as the existing scenarios. The + // stale-owner check after a debug-build correlation adds up to ~10s. + childTimeoutSeconds(150), line -> { if (line.startsWith(LeakTagCorrelationScenario.FOUND_MARKER) || line.equals(LeakTagCorrelationScenario.NOT_FOUND_MARKER) || line.startsWith(LeakTagCorrelationScenario.TAG_OUT_OF_POOL_MARKER) || line.startsWith(LeakTagCorrelationScenario.NO_LIVE_OBJECT_MARKER) || line.startsWith(LeakTagCorrelationScenario.TRANSIENT_CHAIN_MARKER) - || line.startsWith(LeakTagCorrelationScenario.NOISE_CHAIN_MARKER)) { + || line.startsWith(LeakTagCorrelationScenario.NOISE_CHAIN_MARKER) + || line.startsWith(LeakTagCorrelationScenario.NOT_OWNER_VERIFIED_MARKER) + || line.startsWith(LeakTagCorrelationScenario.STALE_OWNER_CHAIN_MARKER)) { resultLine.set(line); } if (line.startsWith("[TEST::INFO]")) { @@ -162,16 +158,21 @@ private void assertCorrelationFound(String pauseTargetOption, && line.contains("state=FRONTIER")) { frontierRotationSeen.set(true); } - testLogTail.addLast(line); - while (testLogTail.size() > 2000) { - testLogTail.removeFirst(); + // The reader thread can still be running after a join timeout, so every access + // to the tail is synchronized (diagnostics() reads a snapshot). + synchronized (testLogTail) { + testLogTail.addLast(line); + while (testLogTail.size() > 2000) { + testLogTail.removeFirst(); + } } } return LineConsumerResult.CONTINUE; }, null); - assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertTrue(result.inTime, "Child process did not exit within the wait timeout" + + diagnostics(testLogTail)); assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); assertNotNull(resultLine.get(), "Child process never printed a recognizable result marker" + " on stdout" + diagnostics(testLogTail)); @@ -179,9 +180,8 @@ private void assertCorrelationFound(String pauseTargetOption, if (requireFrontierRotation) { assertTrue(frontierRotationSeen.get(), "leak-accumulation rotation never selected a FRONTIER-state (un-expanded) " - + "holder under the pod-like tight window - the round-4 regression this " - + "variant exists to catch (tier dead: zero selections) would look exactly " - + "like this" + diagnostics(testLogTail)); + + "holder under the tight window - a dead rotation tier (zero selections) " + + "looks exactly like this" + diagnostics(testLogTail)); } if (line.startsWith(LeakTagCorrelationScenario.FOUND_MARKER)) { long targetTag = Long.parseLong( @@ -193,7 +193,7 @@ private void assertCorrelationFound(String pauseTargetOption, } if (requireFrontierRotation && (line.equals(LeakTagCorrelationScenario.NOT_FOUND_MARKER) || line.startsWith(LeakTagCorrelationScenario.TAG_OUT_OF_POOL_MARKER))) { - // The pod regime legitimately produces these without full correlation completing + // The tight window legitimately produces these without full correlation completing // (see the tight variant's comment); the loose variant still fails on them. return; } @@ -210,7 +210,11 @@ private void assertCorrelationFound(String pauseTargetOption, * gotw batch-size trace (batch control regressions), and any leak-tag interception or * correlation lines. Capped so the message stays readable. */ - private static String diagnostics(Deque testLogTail) { + private static String diagnostics(Deque testLogTailRef) { + List testLogTail; + synchronized (testLogTailRef) { + testLogTail = new ArrayList<>(testLogTailRef); + } if (testLogTail.isEmpty()) { return "\n(no TEST_LOG output captured - non-debug build or no reference-chain passes ran)"; } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index 936f028fd7..7638c3d387 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -6,6 +6,7 @@ package com.datadoghq.profiler.referencechains; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import org.openjdk.jmc.common.IMCType; import org.openjdk.jmc.common.item.IItem; import org.openjdk.jmc.common.item.IItemCollection; @@ -18,9 +19,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; -import java.util.ArrayList; -import java.util.List; +import java.util.Map; + +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_BASE; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_POOL_SIZE; /** * The "leak-tag correlation" use case, run inside a genuinely separate child JVM by @@ -31,9 +35,8 @@ * matching a {@code datadog.HeapLiveObject} event's {@code leakTag} for the same object - * the chain-to-live-heap-sample correlation the whole leak-tag pool exists to provide. * - *

    Each ingredient exists to reproduce, in miniature, a condition that only showed - * against real large deployments before this scenario existed (all observed live on a - * production-like pod, each escaping every earlier local test): + *

    Each ingredient reproduces, in miniature, a condition of large deployments that + * smaller local tests do not reach: *

      *
    • Scale ({@link #FILLER}): ~100k live filler objects grow the JVMTI tag map * enough that {@code GetObjectsWithTags} has a real per-call cost floor, the BFS @@ -52,7 +55,12 @@ * chunk's chain must carry a {@code targetTag} in the leak-tag pool's range - * mirrored here from {@code LEAK_TAG_BASE}/{@code LEAK_TAG_POOL_SIZE} * (referenceChains.h) - and at least one {@code datadog.HeapLiveObject} event for a - * {@code byte[]} must carry the same value in {@code leakTag}.
      • + * {@code byte[]} must carry the same value in {@code leakTag} (joined on + * {@code (tag, leakTagIncarnation)}, since leak tags are recycled). On -DDEBUG builds the + * chain must also be the one {@code getReferenceChainResolvedTargetTagForTest0} resolves for + * a live {@link #LEAK_SINK} chunk (instance identity, not just class), and after that owner + * chunk is dropped and its tag released, its chain must stop being emitted under that + * {@code (tag, leakTagIncarnation)}. *
    * *

    Same separate-process rationale as {@code LeakingCacheScenario}'s own class comment @@ -107,13 +115,6 @@ private static void addChunk() { LEAK_SINK.add(new byte[CHUNK_BYTES]); } - // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE): the pool range a - // correlated chain's targetTag must fall into, and the range a matching HeapLiveObject's - // leakTag must fall into. This is the contract under test, asserted against runtime - // event values - not constants compared to constants. - private static final long LEAK_TAG_BASE = 0x40000000L; - private static final long LEAK_TAG_POOL_SIZE = 256; - /** * Transient handoff channel for the noise representative - deliberately a local of * {@code run()} (captured by the noise thread's lambda), NOT a static field: any static @@ -183,6 +184,22 @@ NoisePayload take() { */ public static final String NOISE_CHAIN_MARKER = "[correlation-noise-chain-emitted] "; + /** + * Printed (with the tag) if no correlated in-pool byte[] chain was the chain the identity seam + * resolves for a live {@link #LEAK_SINK} chunk. -DDEBUG builds only. + */ + public static final String NOT_OWNER_VERIFIED_MARKER = "[correlation-not-owner-verified] "; + + /** + * Printed (with the tag) if, after the verified owner chunk was dropped, every dump still + * emitted a byte[] chain under the owner's {@code (tag, leakTagIncarnation)} - the dead + * owner's cached chain outliving its tag (ownership-stamp regression). -DDEBUG builds only. + */ + public static final String STALE_OWNER_CHAIN_MARKER = "[correlation-stale-owner-chain] "; + + /** Dumps the stale-owner check waits through for the dropped chunk's tag to be released. */ + private static final int STALE_CHECK_ATTEMPTS = 10; + /** * Scan outcome: payload fields from one dump; transientViolations/noiseChains accumulated * across all scans by keepBest(). @@ -190,6 +207,12 @@ NoisePayload take() { private static final class ScanResult { ReferenceChainAssertions.ChainMatch payloadChain; boolean payloadChainInPool; + // In-pool and (on -DDEBUG builds) resolved by the identity seam for a live LEAK_SINK chunk. + boolean payloadChainOwnerVerified; + // LEAK_SINK index of that chunk at scan time; -1 when not verified or on release builds. + int payloadOwnerIndex = -1; + // ReferenceChain.leakTagIncarnation of payloadChain; -1 when the field is absent. + long payloadChainIncarnation = -1; int liveObjectMatchesForChainTag; final List transientViolations = new ArrayList<>(); final List noiseChains = new ArrayList<>(); @@ -219,7 +242,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } // The tid must be captured on THIS thread: the per-(klass, tid) seeding below // qualifies the allocating thread, and the noise instances are all allocated here. - handoff.publish(held[0], JavaProfiler.getTid()); + handoff.publish(held[0], ReferenceChainTestHooks.getTid()); long slot = 0; while (true) { try { @@ -242,7 +265,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD profiler.execute(startCommand); } - boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + boolean debugBuild = ReferenceChainAssertions.isDebugInstrumentedBuild(); int leakTid = 0; int noiseTid = 0; if (debugBuild) { @@ -268,8 +291,8 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD if (handoff.representative == null || handoff.tid <= 0) { throw new IllegalStateException("Noise thread did not publish a representative and tid"); } - JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, LEAK_SINK.get(0)); - JavaProfiler.setKlassPopulationRepresentativeForTest0(NOISE_KLASS_ID, handoff.take()); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, LEAK_SINK.get(0)); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(NOISE_KLASS_ID, handoff.take()); // Per-(klass, tid) qualification seeds: selectLeakCandidates() now also // requires a qualifying ALLOCATING THREAD, and tagLeakInstances() scopes leak // tags to exactly those tids' instances - so the seeded qualification must @@ -278,13 +301,13 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // the tagging scope. Same epoch-aligned ramp shape as the klass seeds // (scaled to fit the per-tid u8 ring); the synthetic flag keeps these // ramps immune to the real fold's absent-tid decay between rounds. - leakTid = JavaProfiler.getTid(); + leakTid = ReferenceChainTestHooks.getTid(); noiseTid = handoff.tid; for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedKlassPopulationSample0(NOISE_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); - JavaProfiler.seedTidTrendSample0(NOISE_KLASS_ID, noiseTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(NOISE_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(NOISE_KLASS_ID, noiseTid, epoch * 3, epoch); } // Maintenance continues the seeded ramp instead of restarting at 1. hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; @@ -292,16 +315,16 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // expansion) before any growth round - same load-bearing wait and rationale as // StaticFieldGrowingCollectionScenario: the late chunks added afterward must be // discovered via rotation re-walks, not by a first pass that has not even run yet. - int initialPasses = JavaProfiler.referenceChainPassesRunForTest0(); + int initialPasses = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); for (int i = 0; i < 300; i++) { - if (JavaProfiler.referenceChainPassesRunForTest0() != initialPasses) { + if (ReferenceChainTestHooks.referenceChainPassesRunForTest0() != initialPasses) { break; } Thread.sleep(100); } - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); } else { - // No debug seams outside debug builds - give the real allocation sampler and the + // No test seams outside -DDEBUG builds (debug/asan/tsan) - give the real allocation sampler and the // pass loop time to notice the seeded population on their own. Thread.sleep(5000); } @@ -322,13 +345,13 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD addChunk(); if (debugBuild) { hysteresisEpoch++; - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( NOISE_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0( + ReferenceChainTestHooks.seedTidTrendSample0( LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0( + ReferenceChainTestHooks.seedTidTrendSample0( NOISE_KLASS_ID, noiseTid, hysteresisEpoch * 3, hysteresisEpoch); } System.gc(); @@ -342,9 +365,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // win the tagging priority. Rounds are still dumped every time so // the final dump always holds the freshest chains. if (round % 3 == 0) { - best = keepBest(best, scan(scratchDumpPath)); - if (best != null && best.payloadChainInPool - && best.liveObjectMatchesForChainTag > 0) { + // The identity seam is read after the dump: a chain in the dump for a live chunk stays + // cached (snapshot-and-keep), and chains built since only add keys the dump lacks. + best = keepBest(best, scan(scratchDumpPath, debugBuild ? liveOwnerTags() : null)); + if (correlated(best)) { break; // full correlation achieved - no need to grow further } } @@ -352,24 +376,24 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } for (int attempt = 0; attempt < 5; attempt++) { - if (best != null && best.payloadChainInPool && best.liveObjectMatchesForChainTag > 0) { + if (correlated(best)) { break; } if (debugBuild) { hysteresisEpoch++; - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( NOISE_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0( + ReferenceChainTestHooks.seedTidTrendSample0( LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0( + ReferenceChainTestHooks.seedTidTrendSample0( NOISE_KLASS_ID, noiseTid, hysteresisEpoch * 3, hysteresisEpoch); } Thread.sleep(1000); System.gc(); profiler.dump(scratchDumpPath); - best = keepBest(best, scan(scratchDumpPath)); + best = keepBest(best, scan(scratchDumpPath, debugBuild ? liveOwnerTags() : null)); } // Gate violations are 'ever emitted' contracts, independent of whether a payload chain @@ -391,13 +415,107 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD System.out.println(TAG_OUT_OF_POOL_MARKER + best.payloadChain.targetTag); return; } + if (!best.payloadChainOwnerVerified) { + System.out.println(NOT_OWNER_VERIFIED_MARKER + best.payloadChain.targetTag); + return; + } if (best.liveObjectMatchesForChainTag == 0) { System.out.println(NO_LIVE_OBJECT_MARKER + best.payloadChain.targetTag); return; } + if (debugBuild) { + long ownedTag = best.payloadChain.targetTag; + long ownedIncarnation = best.payloadChainIncarnation; + // Drop the verified owner: its tracked entry dies, cleanup_table() releases the leak tag + // (stamp bump), and the next acquire may hand the same tag to another object. The dead + // owner's chain must never be emitted under its old (tag, stamp) again. The removed chunk + // is deliberately not kept in a local. + LEAK_SINK.remove(best.payloadOwnerIndex); + boolean clean = false; + for (int attempt = 0; attempt < STALE_CHECK_ATTEMPTS && !clean; attempt++) { + System.gc(); + Thread.sleep(500); + profiler.dump(scratchDumpPath); + clean = !hasByteArrayChain(scratchDumpPath, ownedTag, ownedIncarnation) + // Without the stamp field the tag alone cannot tell a new owner from the dead one. + || (ownedIncarnation < 0 && liveOwnerTags().containsKey(ownedTag)); + } + if (!clean) { + System.out.println(STALE_OWNER_CHAIN_MARKER + ownedTag); + return; + } + } System.out.println(FOUND_MARKER + best.payloadChain.targetTag); } + private static boolean correlated(ScanResult r) { + return r != null && r.payloadChainInPool && r.payloadChainOwnerVerified + && r.liveObjectMatchesForChainTag > 0; + } + + /** + * Leak tag -> LEAK_SINK index of the chunk whose own cached chain carries it, via the identity + * seam (0 means no chain resolved for that chunk). Stores indices, never chunk references, so + * an owner can still die. + */ + private static Map liveOwnerTags() { + Map owners = new HashMap<>(); + for (int i = 0; i < LEAK_SINK.size(); i++) { + long tag = + ReferenceChainTestHooks.getReferenceChainResolvedTargetTagForTest0(LEAK_SINK.get(i)); + if (tag != 0) { + owners.put(tag, i); + } + } + return owners; + } + + /** + * Whether the dump holds a byte[] chain with this targetTag and leakTagIncarnation (-1 matches + * a chain without the field). + */ + private static boolean hasByteArrayChain(Path scratchDumpPath, long tag, long incarnation) + throws Exception { + if (!Files.exists(scratchDumpPath)) { + return false; + } + IItemCollection events; + try (InputStream in = Files.newInputStream(scratchDumpPath)) { + events = JfrLoaderToolkit.loadEvents(in); + } + IItemCollection chains = events.apply( + org.openjdk.jmc.common.item.ItemFilters.type("datadog.ReferenceChain")); + for (IItemIterable iterable : chains) { + IType type = iterable.getType(); + IMemberAccessor chainAccessor = ReferenceChainAssertions.findAccessor(type, "chain"); + IMemberAccessor targetTagAccessor = + ReferenceChainAssertions.findAccessor(type, "targetTag"); + IMemberAccessor incarnationAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); + if (chainAccessor == null || targetTagAccessor == null) { + continue; + } + for (IItem item : iterable) { + Object chainValue = chainAccessor.getMember(item); + if (!(chainValue instanceof Object[])) { + continue; + } + Object[] rawChain = (Object[]) chainValue; + if (rawChain.length == 0 || !(rawChain[0] instanceof IMCType) + || !"byte[]".equals(((IMCType) rawChain[0]).getFullName())) { + continue; + } + long chainIncarnation = incarnationAccessor != null + ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; + if (ReferenceChainAssertions.numberValue(targetTagAccessor.getMember(item)) == tag + && chainIncarnation == incarnation) { + return true; + } + } + } + return false; + } + /** * Payload state comes from the latest scan that saw a payload chain (a scan without one * never displaces one that had it). Gate violations (transient/noise chains) are @@ -426,12 +544,17 @@ private static void mergeDistinct(List into, List from) { } /** - * Scans one dump: the first leaked {@code byte[]} chain (if any), whether its targetTag is - * in the leak-tag pool, how many {@code datadog.HeapLiveObject} events for a - * leaked {@code byte[]} carry that same tag in {@code leakTag}, plus the two gate - * regressions (transient-rooted depth<=1 chains, any {@link NoisePayload} chain). + * Scans one dump: the best leaked {@code byte[]} chain (if any), whether its targetTag is + * in the leak-tag pool and owned by a live LEAK_SINK chunk, how many + * {@code datadog.HeapLiveObject} events for a leaked {@code byte[]} carry that same + * {@code (leakTag, leakTagIncarnation)}, plus the two gate regressions (transient-rooted + * depth<=1 chains, any {@link NoisePayload} chain). + * + * @param liveOwners {@link #liveOwnerTags()} read after this dump, or null on release builds + * (no identity seam: any in-pool chain counts as owner-verified) */ - private static ScanResult scan(Path scratchDumpPath) throws Exception { + private static ScanResult scan(Path scratchDumpPath, Map liveOwners) + throws Exception { if (!Files.exists(scratchDumpPath)) { return null; } @@ -451,6 +574,8 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { IMemberAccessor depthAccessor = ReferenceChainAssertions.findAccessor(type, "depth"); IMemberAccessor rootKindAccessor = ReferenceChainAssertions.findAccessor(type, "rootKind"); + IMemberAccessor incarnationAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); for (IItem item : iterable) { Object chainValue = chainAccessor.getMember(item); if (!(chainValue instanceof Object[])) { @@ -478,9 +603,14 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { // instance of the candidate class gets one, and only the tracked-and- // tagged instances carry a pool-range targetTag - the rest legitimately // carry their frontier tag (the correlation is additive on top of the - // chain, not a property of every chain). Prefer the first in-pool - // (correlated) chain over any out-of-pool one seen earlier. - if ((inPool && !result.payloadChainInPool) || result.payloadChain == null) { + // chain, not a property of every chain). Prefer the first owner-verified + // in-pool chain, then any in-pool one, over an out-of-pool one seen earlier. + boolean ownerVerified = + inPool && (liveOwners == null || liveOwners.containsKey(targetTag)); + int score = inPool ? (ownerVerified ? 2 : 1) : 0; + int currentScore = result.payloadChain == null ? -1 + : result.payloadChainInPool ? (result.payloadChainOwnerVerified ? 2 : 1) : 0; + if (score > currentScore) { List chain = new ArrayList<>(rawChain.length); for (Object element : rawChain) { chain.add((IMCType) element); @@ -488,6 +618,11 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { result.payloadChain = new ReferenceChainAssertions.ChainMatch( chain, targetTag, depth); result.payloadChainInPool = inPool; + result.payloadChainOwnerVerified = ownerVerified; + result.payloadOwnerIndex = + ownerVerified && liveOwners != null ? liveOwners.get(targetTag) : -1; + result.payloadChainIncarnation = incarnationAccessor != null + ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; } } } @@ -502,10 +637,16 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { ReferenceChainAssertions.findAccessor(type, "leakTag"); IMemberAccessor objectClassAccessor = ReferenceChainAssertions.findAccessor(type, "objectClass"); + IMemberAccessor incarnationAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); for (IItem item : iterable) { long leakTag = leakTagAccessor != null ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; - if (leakTag != result.payloadChain.targetTag) { + long incarnation = incarnationAccessor != null + ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; + // Leak tags are recycled: the tag alone is ambiguous across chunks. + if (leakTag != result.payloadChain.targetTag + || incarnation != result.payloadChainIncarnation) { continue; } Object objectClass = objectClassAccessor != null diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java index 3f7eec2b04..7d91bdc90e 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java @@ -6,6 +6,7 @@ package com.datadoghq.profiler.referencechains; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import org.openjdk.jmc.common.IMCType; import org.openjdk.jmc.common.item.IItemCollection; import org.openjdk.jmc.common.item.ItemFilters; @@ -94,7 +95,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD } System.out.println("[debug] profiler.execute() returned"); - boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + boolean debugBuild = ReferenceChainAssertions.isDebugInstrumentedBuild(); boolean seededTestKlassTrend = false; ReferenceChainAssertions.ChainMatch match = null; int totalRounds = 25; @@ -121,19 +122,19 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // Representative BEFORE seeding: setKlassPopulationRepresentativeForTest0 aliases the // synthetic id to the representative's real klass id, so the seeds below land in the // real population entry (candidate matching is keyed by real id). - JavaProfiler.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get("seed-0")); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get("seed-0")); if (!seededTestKlassTrend) { // The qualifying tid is THIS thread's - the creator of "seed-0". The per-round // allocator threads are one-shot, so none of their tids carries a sustained // per-tid signal. - int leakTid = JavaProfiler.getTid(); + int leakTid = ReferenceChainTestHooks.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedTidTrendSample0(CACHED_PAYLOAD_TEST_KLASS_ID, leakTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(CACHED_PAYLOAD_TEST_KLASS_ID, leakTid, epoch * 3, epoch); } seededTestKlassTrend = true; } - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); profiler.dump(scratchDumpPath); match = findMatch(scratchDumpPath); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java new file mode 100644 index 0000000000..5147ec909e --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java @@ -0,0 +1,131 @@ +/* + * Copyright 2026, Datadog, Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.referencechains; + +import com.datadoghq.profiler.ReferenceChainTestHooks; + +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; + +/** + * A single long-lived thread that runs caller-supplied allocation batches on demand. + * + *

    Tests use one persistent thread so every instance of the leak klass is allocated by the + * same thread: a fresh short-lived thread per round gets a new tid each time, and the per-tid + * trend qualification ages out before the population trend can establish itself (the flake the + * {@code @Order(1)} test documents). Callers must also seed with this thread's tid, obtained + * from {@link #tid()} - captured on the allocator thread itself, never the caller's. + * + *

    {@link #run(Runnable)} blocks until the batch completes and rethrows any failure from the + * allocator thread, so allocations are finished (and any exception surfaced) before the caller + * proceeds to {@code System.gc()} or a dump. + */ +final class PersistentAllocator implements AutoCloseable { + // Poison task handed to the worker by close(); identity compared, never run. + private static final Runnable CLOSE = () -> { }; + + private static final long HANDOFF_TIMEOUT_SECONDS = 30; + + private final SynchronousQueue handoff = new SynchronousQueue<>(); + // Worker-to-caller completion slot: null = success, Throwable = the batch's failure. + private final SynchronousQueue completion = new SynchronousQueue<>(); + private final Thread thread; + private volatile int tid; + private volatile boolean closed; + + PersistentAllocator(String name) { + thread = new Thread(this::loop, name); + // Daemon: a test that dies before close() must not hold the JVM open on handoff.take(). + thread.setDaemon(true); + thread.start(); + // The caller may call tid() immediately after construction (seed-before-wait ordering); + // do not return until the allocator thread has published it. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(HANDOFF_TIMEOUT_SECONDS); + while (tid == 0 && thread.isAlive()) { + if (System.nanoTime() - deadline > 0) { + throw new IllegalStateException("allocator thread " + name + " never published its tid"); + } + Thread.yield(); + } + if (tid == 0) { + throw new IllegalStateException("allocator thread " + name + " died before publishing its tid"); + } + } + + /** The allocator thread's OS tid, captured by that thread itself at startup. */ + int tid() { + return tid; + } + + /** + * Runs {@code task} on the allocator thread, blocking until it completes. A task that throws + * is rethrown on the caller's thread, so a failed batch fails the test instead of stalling + * the next round on partially-allocated state. + */ + void run(Runnable task) { + if (closed || !thread.isAlive()) { + throw new IllegalStateException("allocator thread " + thread.getName() + " is no longer running"); + } + try { + if (!handoff.offer(task, HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new IllegalStateException("allocator thread " + thread.getName() + " did not accept a task"); + } + Object outcome = completion.poll(HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (outcome == null) { + throw new IllegalStateException("allocator thread " + thread.getName() + " never finished a task"); + } + if (outcome instanceof RuntimeException) { + throw (RuntimeException) outcome; + } + if (outcome instanceof Error) { + throw (Error) outcome; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for allocator thread " + thread.getName(), e); + } + } + + /** Stops the allocator thread and waits for it to exit. Idempotent. */ + @Override + public void close() { + if (closed) { + return; + } + closed = true; + try { + if (thread.isAlive() && handoff.offer(CLOSE, HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + // Drain the exit acknowledgement so close() returns only after the thread is done. + completion.poll(HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + thread.join(TimeUnit.SECONDS.toMillis(HANDOFF_TIMEOUT_SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void loop() { + tid = ReferenceChainTestHooks.getTid(); + try { + while (true) { + Runnable task = handoff.take(); + if (task == CLOSE) { + completion.offer(new Object()); // exit acknowledgement; no further work arrives + return; + } + Object outcome = null; + try { + task.run(); + } catch (RuntimeException | Error t) { + outcome = t; + } + completion.offer(outcome != null ? outcome : new Object()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java index e77f1594a9..dce3a2d76b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java @@ -31,6 +31,26 @@ public final class ReferenceChainAssertions { private ReferenceChainAssertions() {} + /** + * Leak-tag pool range, mirrored from referenceChains.h / livenessTracker.h (LEAK_TAG_BASE, + * LEAK_TAG_POOL_SIZE): a correlated chain's targetTag and its HeapLiveObject's leakTag must fall + * into [LEAK_TAG_BASE, LEAK_TAG_BASE + LEAK_TAG_POOL_SIZE). Asserted against runtime event + * values. + */ + public static final long LEAK_TAG_BASE = 0x40000000L; + + public static final long LEAK_TAG_POOL_SIZE = 256; + + /** + * True on every native build that compiles javaApi.cpp's {@code #ifdef DEBUG} test seams: debug, + * asan and tsan (all add -DDEBUG, ConfigurationPresets.kt). Release builds get no-op seam stubs. + * Child JVMs see the same answer because the launchers forward {@code -Dddprof_test.config}. + */ + public static boolean isDebugInstrumentedBuild() { + String config = System.getProperty("ddprof_test.config"); + return config != null && ("debug".equals(config) || config.endsWith("san")); + } + /** Result of {@link #findMatchForClass(IItemCollection, Class)}: one resolved chain event's fields. */ public static final class ChainMatch { public final List chain; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java index 099063d9a1..9440885bd3 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java @@ -51,22 +51,19 @@ public class ReferenceChainJfrParserTest { private static final String EVENT_TYPE = "datadog.ReferenceChain"; - /** - * Must match the companion gtest's {@code chainRoundtripJfrPath()} (TMPDIR, else /tmp): the - * two run in different Gradle modules with no other shared filesystem contract. - */ - private static Path roundtripJfrPath() { - String tmp = System.getenv("TMPDIR"); - String dir = (tmp != null && !tmp.isEmpty()) ? tmp : "/tmp"; - return Paths.get(dir, "datadog_reference_chain_roundtrip.jfr"); - } + /** Set by the build for both this test and the companion gtest (see build.gradle.kts). */ + private static final String ROUNDTRIP_JFR_ENV = "DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR"; @Test public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() throws IOException, CouldNotLoadRecordingException { - Path jfrPath = roundtripJfrPath(); + String configured = System.getenv(ROUNDTRIP_JFR_ENV); + Assumptions.assumeTrue(configured != null && !configured.isEmpty(), + ROUNDTRIP_JFR_ENV + " is not set - this test only runs from the Gradle test tasks"); + Path jfrPath = Paths.get(configured); // Assumption, not assertion: the producing gtest does not run on CI runners without Google - // Test (e.g. the el7 testRelease job), where the file can never exist. + // Test (e.g. the el7 testRelease job) or with -Pskip-gtest. The build deletes the file before + // the gtest runs, and when the gtest is skipped, so an existing file is always this build's. Assumptions.assumeTrue(Files.exists(jfrPath), "Expected " + jfrPath + " to exist - run " + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " @@ -81,6 +78,7 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() List resolvedChain = null; long targetTag = -1; + long leakTagIncarnation = -1; int depth = -1; for (IItemIterable iterable : chainEvents) { IType type = iterable.getType(); @@ -88,6 +86,9 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() IMemberAccessor edgesAccessor = findAccessor(type, "edges"); IMemberAccessor targetTagAccessor = findAccessor(type, "targetTag"); IMemberAccessor depthAccessor = findAccessor(type, "depth"); + IMemberAccessor incarnationAccessor = findAccessor(type, "leakTagIncarnation"); + assertNotNull(incarnationAccessor, "No accessor for 'leakTagIncarnation' field on " + + EVENT_TYPE); assertNotNull(chainAccessor, "No accessor for 'chain' field on " + EVENT_TYPE); assertNotNull(edgesAccessor, "No accessor for 'edges' field on " + EVENT_TYPE + " - the new retention-edge label array (T_STRING|F_ARRAY) did not make it " @@ -137,6 +138,7 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() targetTag = ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); } } + leakTagIncarnation = longValue(incarnationAccessor.getMember(item)); if (depthAccessor != null) { Object v = depthAccessor.getMember(item); if (v instanceof Number) { @@ -162,6 +164,17 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() assertEquals("com.test.ChainRoot", resolvedChain.get(2).getFullName()); assertEquals(3L, targetTag, "targetTag should be the leaf's frontier tag (3)"); assertEquals(2, depth, "depth should be the leaf's own depth (2 hops from the root)"); + assertEquals(5L, leakTagIncarnation, + "leakTagIncarnation should carry the event's ownership stamp (5)"); + } + + private static long longValue(Object v) { + if (v instanceof Number) { + return ((Number) v).longValue(); + } else if (v instanceof org.openjdk.jmc.common.unit.IQuantity) { + return ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); + } + return -1; } private static IMemberAccessor findAccessor(IType type, String identifier) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java index 57107cddaf..d50f79c647 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTestSeamsTest.java @@ -7,6 +7,7 @@ import com.datadoghq.profiler.AbstractProfilerTest; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import com.datadoghq.profiler.Platform; import org.junit.jupiter.api.Test; @@ -26,8 +27,8 @@ * dependency on {@code selectLeakCandidates()} organically picking the right klass. * * - *

    Only runs under the debug native build - the backing native methods do not exist in a - * release build (javaApi.cpp's {@code #ifdef DEBUG} guard). + *

    Only runs under -DDEBUG native builds (debug/asan/tsan) - the backing native methods are + * no-op stubs in a release build (javaApi.cpp's {@code #ifdef DEBUG} guard). */ public class ReferenceChainTestSeamsTest extends AbstractProfilerTest { @@ -64,7 +65,7 @@ protected boolean isPlatformSupported() { } private static void assumeDebugBuild() { - assumeTrue("debug".equals(System.getProperty("ddprof_test.config"))); + assumeTrue(ReferenceChainAssertions.isDebugInstrumentedBuild()); } /** @@ -77,20 +78,20 @@ private static void assumeDebugBuild() { @Test public void shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope() { assumeDebugBuild(); - JavaProfiler.resetKlassPopulationForTest0(); - JavaProfiler.setGcGenerationsEnabled0(true); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.setGcGenerationsEnabled0(true); // KLASS_POPULATION_MIN_FILL_FOR_TREND = 10 only makes a trend eligible; selectLeakCandidates() // also needs LEAK_TREND_HYSTERESIS_BASE = 5 consecutive qualifying epochs, so at least // 10 + (5 - 1) = 14 increasing samples are needed; 20 gives headroom. for (int epoch = 1; epoch <= 20; epoch++) { - JavaProfiler.seedKlassPopulationSample0(SLOPE_TEST_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(SLOPE_TEST_KLASS_ID, epoch * 10, epoch); // selectLeakCandidates() also requires a qualifying allocating thread. No tracked instance // needs to match here, so a fixed synthetic tid suffices. - JavaProfiler.seedTidTrendSample0(SLOPE_TEST_KLASS_ID, 4242, epoch * 3, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(SLOPE_TEST_KLASS_ID, 4242, epoch * 3, epoch); } - int[] candidates = JavaProfiler.selectLeakCandidateKlassIds0(); + int[] candidates = ReferenceChainTestHooks.selectLeakCandidateKlassIds0(); boolean found = false; for (int klassId : candidates) { if (klassId == SLOPE_TEST_KLASS_ID) { @@ -108,18 +109,20 @@ public void shouldSelectSeededKlassAsLeakCandidateOnPositiveSlope() { * seeded leak candidate's representative, then drives one BFS pass and one poll cycle * synchronously. This is the "trigger the refchain on a known live heap sample" seam: it * proves {@code runPass()}/{@code pollWatchedTargets()}/{@code buildChainEvent()} correctly - * produce a chain event for a target this test controls directly, decoupled from whether - * LivenessTracker's probabilistic sampler would have picked the same object on its own. + * produce a chain event whose leaf is this test's {@link ChainTarget}, decoupled from whether + * LivenessTracker's probabilistic sampler would have picked the same object on its own. Only + * chains with a {@code ChainTarget} leaf count, so a chain another test left cached in this + * shared JVM cannot satisfy the assertion. */ @Test public void shouldReconstructChainForDirectlyTaggedRoot() { assumeDebugBuild(); - JavaProfiler.resetKlassPopulationForTest0(); - JavaProfiler.setGcGenerationsEnabled0(true); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.setGcGenerationsEnabled0(true); // Guards against inheriting a full/tiny FrontierTable from an earlier test in this // no-forkEvery JVM (e.g. ReferenceChainTrackingTest's framecap=1 abandonment test), against // which tagAsReferenceChainRoot0()'s insert() could fail. - JavaProfiler.resetReferenceChainSearchForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); // Referenced only through the static holder, never a local: a local would make the target a // stack-local GC root for the whole test body, and the noise gate suppresses chains rooted in @@ -128,23 +131,23 @@ public void shouldReconstructChainForDirectlyTaggedRoot() { // Representative before seeding: setting it registers the synthetic->real id alias, so the // seeds land under the target's real class id. - JavaProfiler.setKlassPopulationRepresentativeForTest0(CHAIN_TEST_KLASS_ID, chainTargetHolder); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(CHAIN_TEST_KLASS_ID, chainTargetHolder); // 20 epochs to clear the hysteresis gate - see the slope test above. for (int epoch = 1; epoch <= 20; epoch++) { - JavaProfiler.seedKlassPopulationSample0(CHAIN_TEST_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(CHAIN_TEST_KLASS_ID, epoch * 10, epoch); // This thread's real tid, since the target was allocated here. Belt-and-braces: the // chain's primary path is the directly-tagged root plus the seeded representative. - JavaProfiler.seedTidTrendSample0( - CHAIN_TEST_KLASS_ID, JavaProfiler.getTid(), epoch * 3, epoch); + ReferenceChainTestHooks.seedTidTrendSample0( + CHAIN_TEST_KLASS_ID, ReferenceChainTestHooks.getTid(), epoch * 3, epoch); } // Poll before the direct tag: it admits the seeded klass as a watched candidate, and // tagAsRootForTest's recordDiscoveredInstance only lands when a candidate slot already // watches the klass. - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); - long tag = JavaProfiler.tagAsReferenceChainRoot0(chainTargetHolder); + long tag = ReferenceChainTestHooks.tagAsReferenceChainRoot0(chainTargetHolder); assertTrue(tag > 0, "Expected tagAsReferenceChainRoot0 to assign a valid frontier tag"); // One pass is not enough: admitStaticFieldRoots() sweeps loaded classes in budget-bounded @@ -154,15 +157,15 @@ public void shouldReconstructChainForDirectlyTaggedRoot() { // pollWatchedTargetsSerialized) keeps the BFS thread from racing these calls. int eventCount = 0; for (int pass = 0; pass < 40 && eventCount == 0; pass++) { - boolean sawPassRun = JavaProfiler.runReferenceChainPass0(); + boolean sawPassRun = ReferenceChainTestHooks.runReferenceChainPass0(); assertTrue(sawPassRun || pass > 0, "Expected the first runReferenceChainPass0 to run (reference chains enabled)"); - JavaProfiler.pollReferenceChainTargets0(); - eventCount = JavaProfiler.drainReferenceChainEventCount0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); + eventCount = ReferenceChainTestHooks.drainReferenceChainEventCountForLeaf0(chainTargetHolder); } assertTrue(eventCount > 0, - "Expected pollWatchedTargets() to have queued at least one chain event for the " - + "directly-tagged, seeded-representative target"); + "Expected pollWatchedTargets() to have queued at least one chain event whose leaf is " + + "ChainTarget (the directly-tagged, seeded-representative target)"); chainTargetHolder = null; // shared-JVM hygiene } } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java index 0d057ce48c..84d01c6e9b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java @@ -6,7 +6,7 @@ package com.datadoghq.profiler.referencechains; import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import com.datadoghq.profiler.JfrEvent; import com.datadoghq.profiler.JfrEvents; import com.datadoghq.profiler.Platform; @@ -15,12 +15,11 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; -import org.junitpioneer.jupiter.RetryingTest; import java.nio.file.Files; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.TimeUnit; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -66,10 +65,6 @@ public class ReferenceChainTrackingTest extends AbstractProfilerTest { // population reachable for later tests in this shared JVM. private static List gcRootHolder; - // Volatile: handed to the allocator thread every round. The request queue already provides - // happens-before; this keeps the cross-thread visibility explicit. - private static volatile List currentBatch; - // consecutive_positive only starts incrementing once ring_fill reaches // KLASS_POPULATION_MIN_FILL_FOR_TREND=10 (ringThirdsStats()'s fill < min_fill guard resets it to // 0 before that), and selectLeakCandidates() needs it at LEAK_TREND_HYSTERESIS_BASE=5 (or 3 when @@ -169,7 +164,7 @@ protected boolean isPlatformSupported() { * {@code datadog.ReferenceChain} JFR setting (flightRecorder.cpp's * {@code writeBoolSetting(buf, T_REFERENCE_CHAIN, "enabled", args._reference_chains)}). */ - @RetryingTest(5) + @Test public void shouldExposeReferenceChainsSettingWhenEnabled() { stopProfiler(); JfrEvents settings = verifyEvents("jdk.ActiveSetting"); @@ -208,46 +203,18 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { // whichever array the walk snapshots is non-empty. gcRootHolder = new ArrayList<>(); gcRootHolder.add(new ChainLink("gc-root-seed")); - if ("debug".equals(System.getProperty("ddprof_test.config"))) { + if (ReferenceChainAssertions.isDebugInstrumentedBuild()) { // @Order(1) only orders this class: another test class in this no-forkEvery JVM may have // left the singleton search non-RUNNING, after which runPass() never takes its root-seeded // walk again. The population reset is needed for the same reason: _klass_population // survives stop()/start(), and a leftover klass already past the hysteresis gate would let // the first pass start regardless of this method's own seeding, masking whether it is // sufficient. Debug-only: these native seams do not exist in a release build. - JavaProfiler.resetKlassPopulationForTest0(); - JavaProfiler.resetReferenceChainSearchForTest0(); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); } - // One persistent allocator thread, reused by every round: SampledObjectAlloc does not fire - // for this JUnit worker thread's own allocations, but does for threads created after - // Profiler::start(). Keeping a single tid lets the per-(klass, tid) qualification trend - // accumulate real samples every round (foldKlassCountsLocked()), so qualification cannot - // age out mid-test the way a fresh short-lived tid per round did (slope decayed to ~0 at - // ring_fill=22 and the candidate dropped out with the crawl unfinished). - LinkedBlockingQueue allocRequests = new LinkedBlockingQueue<>(); - Semaphore roundDone = new Semaphore(0); - AtomicLong allocatorTid = new AtomicLong(-1); - Thread allocator = new Thread(() -> { - allocatorTid.set(JavaProfiler.getTid()); - try { - Integer requested; - while ((requested = allocRequests.take()) != null) { - List batch = currentBatch; - for (int i = 0; i < requested; i++) { - batch.add(new ChainLink("leak-" + requested + "-" + i)); - } - roundDone.release(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); // test shutdown - see finally below - } - }, "togcroot-leak-allocator"); - allocator.start(); - // The seeding block below needs the allocator's tid; the thread publishes - // its own tid as its first action, so this wait is bounded by thread start. - while (allocatorTid.get() == -1) { - Thread.sleep(1); - } + // Every ChainLink is allocated by one persistent thread - see PersistentAllocator. + PersistentAllocator allocator = new PersistentAllocator("togcroot-leak-allocator"); Path scratchDumpPath = Paths.get("referencechains-population-scratch.jfr"); try { @@ -271,18 +238,17 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { for (int round = 1; round <= totalRounds && match == null; round++) { int newInstances = Math.min(round, 10) * 600; List newLinks = new ArrayList<>(newInstances); - currentBatch = newLinks; // published before the request - see the allocator's loop - if (!allocRequests.offer(newInstances)) { - throw new IllegalStateException( - "allocRequests queue rejected instance count " + newInstances); - } - roundDone.acquireUninterruptibly(); + allocator.run(() -> { + for (int i = 0; i < newInstances; i++) { + newLinks.add(new ChainLink("leak-" + newInstances + "-" + i)); + } + }); gcRootHolder.addAll(newLinks); System.gc(); dump(scratchDumpPath); match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); - if (match == null && "debug".equals(System.getProperty("ddprof_test.config"))) { + if (match == null && ReferenceChainAssertions.isDebugInstrumentedBuild()) { // Under generations=true, shouldRunPass() gates even the first pass on hasLeakSignal(), // which this early falls through to selectLeakCandidates()'s hysteresis gate for some // klass. Seeding SEED_EPOCHS_FOR_HYSTERESIS epochs clears it, authorizing the BFS @@ -296,22 +262,22 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { // Representative before seeding: setting it aliases the synthetic id to the // representative's real klass id, so the seeds land in the real population entry // (seeds under a bare synthetic id satisfy only hasLeakSignal()'s candidate count). - JavaProfiler.setKlassPopulationRepresentativeForTest0(CHAIN_LINK_TEST_KLASS_ID, gcRootHolder.get(0)); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(CHAIN_LINK_TEST_KLASS_ID, gcRootHolder.get(0)); if (!seededTestKlassTrend) { // The allocator thread owns every ChainLink allocation, so its tid is the qualifying // one; the seeds accelerate the organic per-tid growth it produces every round. - int leakTid = (int) allocatorTid.get(); + int leakTid = allocator.tid(); // Seed counts match the organic ramp (+1 per epoch): hasQualifyingGrowth() compares // third-window means over seeds followed by real folds, and inflated seeds (epoch*10) // made the slope go negative (-20..-40 observed) once real samples displaced them. // With matched scale ([1..15, 1, 2, 3, ...]) every window pairing stays positive. for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(CHAIN_LINK_TEST_KLASS_ID, epoch, epoch); - JavaProfiler.seedTidTrendSample0(CHAIN_LINK_TEST_KLASS_ID, leakTid, epoch, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(CHAIN_LINK_TEST_KLASS_ID, epoch, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(CHAIN_LINK_TEST_KLASS_ID, leakTid, epoch, epoch); } seededTestKlassTrend = true; } - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); dump(scratchDumpPath); match = ReferenceChainAssertions.findMatchForClass(verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false), ChainLink.class); } @@ -350,11 +316,7 @@ public void shouldReconstructReferrerChainToGcRoot() throws Exception { assertTrue(!gcRootHolder.isEmpty()); // keeps every allocated ChainLink reachable until here } finally { gcRootHolder = null; // shared-JVM hygiene - see the static holder's own comment - currentBatch = null; - // A leftover allocator thread would keep its tid trend alive into later tests in this - // shared, no-forkEvery JVM. - allocator.interrupt(); - allocator.join(1000); + allocator.close(); Files.deleteIfExists(scratchDumpPath); } } @@ -377,15 +339,17 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc // gcRootHolder's seeding: HashMap's table is only allocated on the first put(). Map cache = new HashMap<>(); cache.put("cache-leak-seed", new CachedPayload("cache-leak-seed")); - if ("debug".equals(System.getProperty("ddprof_test.config"))) { + if (ReferenceChainAssertions.isDebugInstrumentedBuild()) { // The natural restart after the @Order(1) search ends is gated on canAffordNewSearch() and // not guaranteed within this method's retry budget, so force a fresh search. The population // reset is also needed: resetReferenceChainSearchForTest0() leaves LivenessTracker's rings // alone, and ChainLink's leftover positive slope could outrank CachedPayload. // Debug-only: these native seams do not exist in a release build. - JavaProfiler.resetKlassPopulationForTest0(); - JavaProfiler.resetReferenceChainSearchForTest0(); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); } + // Every CachedPayload is allocated by one persistent thread - see PersistentAllocator. + PersistentAllocator allocator = new PersistentAllocator("cache-leak-allocator"); Path scratchDumpPath = Paths.get("referencechains-cache-leak-scratch.jfr"); try { // Same round-growth/retry shape (and caps) as shouldReconstructReferrerChainToGcRoot(). @@ -411,33 +375,34 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc // lower rate let CachedPayload's ring stall short of KLASS_POPULATION_MIN_FILL_FOR_TREND. int newEntries = Math.min(round, 10) * 600; int keyOffset = nextKey; - Thread allocator = new Thread(() -> { + allocator.run(() -> { for (int i = 0; i < newEntries; i++) { String key = keys[keyOffset + i]; cache.put(key, new CachedPayload(key)); } }); - allocator.start(); - allocator.join(); nextKey += newEntries; System.gc(); dump(scratchDumpPath); JfrEvents events1 = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); match = ReferenceChainAssertions.findMatchForClass(events1, CachedPayload.class); - if (match == null && "debug".equals(System.getProperty("ddprof_test.config"))) { + if (match == null && ReferenceChainAssertions.isDebugInstrumentedBuild()) { // Same seed-once, representative-first, best-effort-poll rationale as // shouldReconstructReferrerChainToGcRoot(). - JavaProfiler.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get(keys[0])); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(CACHED_PAYLOAD_TEST_KLASS_ID, cache.get(keys[0])); if (!seededTestKlassTrend) { - int leakTid = JavaProfiler.getTid(); + // The allocator thread owns every CachedPayload allocation, and the seeds use the + // organic ramp's scale - both for the reasons shouldReconstructReferrerChainToGcRoot() + // gives. + int leakTid = allocator.tid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedTidTrendSample0(CACHED_PAYLOAD_TEST_KLASS_ID, leakTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(CACHED_PAYLOAD_TEST_KLASS_ID, epoch, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(CACHED_PAYLOAD_TEST_KLASS_ID, leakTid, epoch, epoch); } seededTestKlassTrend = true; } - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); dump(scratchDumpPath); JfrEvents events2 = verifyEvents(scratchDumpPath, "datadog.ReferenceChain", false); match = ReferenceChainAssertions.findMatchForClass(events2, CachedPayload.class); @@ -484,6 +449,7 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc + "storage (cache is a HashMap) - chain was: " + match.chain); assertTrue(!cache.isEmpty()); // keeps every cached CachedPayload reachable until here } finally { + allocator.close(); Files.deleteIfExists(scratchDumpPath); } } @@ -518,7 +484,7 @@ public void shouldReconstructReferrerChainThroughUnboundedCacheLeak() throws Exc @Test @Order(3) public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { - if ("debug".equals(System.getProperty("ddprof_test.config"))) { + if (ReferenceChainAssertions.isDebugInstrumentedBuild()) { // GC + throwaway dump before seeding - see javadoc. System.gc(); Path warmupDumpPath = Paths.get("referencechains-abandoned-test-warmup.jfr"); @@ -527,14 +493,14 @@ public void shouldReportAbandonedSearchOnTinyFrontierCap() throws Exception { } finally { Files.deleteIfExists(warmupDumpPath); } - JavaProfiler.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); // Gate-only seeding: no tagged instance needs to match, so any tid would do. - int abandonTid = JavaProfiler.getTid(); + int abandonTid = ReferenceChainTestHooks.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(ABANDON_TEST_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedTidTrendSample0(ABANDON_TEST_KLASS_ID, abandonTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(ABANDON_TEST_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(ABANDON_TEST_KLASS_ID, abandonTid, epoch * 3, epoch); } - JavaProfiler.resetReferenceChainSearchForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); } List gcRootHolder = new ArrayList<>(); gcRootHolder.add(new ChainLink("middle", new ChainLink("leaf"))); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java index 982864ba03..5f5e3f2fb6 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/StaticFieldGrowingCollectionScenario.java @@ -6,6 +6,7 @@ package com.datadoghq.profiler.referencechains; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import org.openjdk.jmc.common.item.IItemCollection; import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; @@ -82,10 +83,10 @@ private StaticFieldGrowingCollectionScenario() {} SEED_CHUNKS + 1 + TOTAL_ROUNDS + TAIL_ATTEMPTS; // FrontierEntryState::EXPANDED (referenceChainFrontier.h), as reported by - // JavaProfiler.getReferenceChainFrontierStateForTest0(). + // ReferenceChainTestHooks.getReferenceChainFrontierStateForTest0(). private static final int FRONTIER_STATE_EXPANDED = 1; - // JavaProfiler.getReferenceChainFrontierStateForTest0()'s "no frontier entry" sentinel. + // ReferenceChainTestHooks.getReferenceChainFrontierStateForTest0()'s "no frontier entry" sentinel. private static final int FRONTIER_STATE_NONE = -1; /** Printed to stdout, followed by the matched leaf class's name, on success. */ @@ -119,22 +120,20 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // (javaApi.cpp's #ifdef DEBUG block; ConfigurationPresets.kt) - so the seam path must be // enabled on all three. Same predicate as LeakTagCorrelationReferenceChainTest's rotation // gate; release builds get the no-op stubs. - String testConfig = System.getProperty("ddprof_test.config"); - boolean debugBuild = testConfig != null - && ("debug".equals(testConfig) || testConfig.endsWith("san")); + boolean debugBuild = ReferenceChainAssertions.isDebugInstrumentedBuild(); System.out.println(SEAM_PATH_MARKER + debugBuild); int hysteresisEpoch = 0; int leakTid = 0; // Snapshotted BEFORE seeding anything, while the BFS thread's search has not run a single // pass yet (referenceChainPassesRunForTest0() reads 0 here) - see the wait loop below for why // this exact value, not just "nonzero", is what the wait needs to advance past. - int initialPasses = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; + int initialPasses = debugBuild ? ReferenceChainTestHooks.referenceChainPassesRunForTest0() : -1; if (debugBuild) { // Pins the frontier-state seam's -1 sentinel: an untagged object (tag 0) has no frontier // entry. Probed before any seeding - no pass can run until shouldRunPass() sees a leak // signal - so the BFS root walk cannot have tagged the probe yet. - int untaggedState = JavaProfiler.getReferenceChainFrontierStateForTest0(new Object()); + int untaggedState = ReferenceChainTestHooks.getReferenceChainFrontierStateForTest0(new Object()); if (untaggedState != FRONTIER_STATE_NONE) { throw new IllegalStateException("frontier-state seam returned " + untaggedState + " for an untagged probe object, expected " + FRONTIER_STATE_NONE); @@ -164,17 +163,17 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // 10/10 searches), so lateChunk's chain is rebuilt once per search generation regardless. // The seed chunk is held strongly by the static field, so the tracker's weak // representative ref stays live. - JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, LEAK_BUFFER.get(0)); // Seed hysteresis-qualifying epochs so hasLeakSignal()/shouldRunPass() authorize the BFS // thread's first pass. Must precede the passesRun() wait below: a pass only runs once // shouldRunPass() sees a leak signal, so waiting first deadlocks. Per-(klass, tid) seeds // (see LeakTagCorrelationScenario): the qualifying tid must be the allocating thread, and // this thread allocates every LEAK_BUFFER chunk. - leakTid = JavaProfiler.getTid(); + leakTid = ReferenceChainTestHooks.getTid(); for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(LEAK_BUFFER_TEST_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedTidTrendSample0(LEAK_BUFFER_TEST_KLASS_ID, leakTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(LEAK_BUFFER_TEST_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(LEAK_BUFFER_TEST_KLASS_ID, leakTid, epoch * 3, epoch); } hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; } @@ -219,7 +218,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(45); boolean holdersExpanded = false; while (!holdersExpanded && System.nanoTime() < deadline) { - holdersExpanded = JavaProfiler.referenceChainPassesRunForTest0() != initialPasses + holdersExpanded = ReferenceChainTestHooks.referenceChainPassesRunForTest0() != initialPasses && leakBufferHoldersExpanded(backingListField, elementDataField); if (!holdersExpanded) { Thread.sleep(100); @@ -228,7 +227,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD if (!holdersExpanded) { throw new IllegalStateException( "LEAK_BUFFER's holders never all reached EXPANDED within 45s (passesRun=" - + JavaProfiler.referenceChainPassesRunForTest0() + ", initial=" + initialPasses + + ReferenceChainTestHooks.referenceChainPassesRunForTest0() + ", initial=" + initialPasses + ", frontier states [list, backing ArrayList, elementData]=" + describeHolderStates(backingListField, elementDataField) + ") - cannot safely create the chunk this scenario is supposed to discover only " @@ -240,7 +239,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // Snapshotted right after the wait above confirms the holders are EXPANDED - any // later change (advance, or reset-to-lower via a restart) proves a walk that started strictly // after lateChunk existed. - int passesRunBeforeLateChunk = debugBuild ? JavaProfiler.referenceChainPassesRunForTest0() : -1; + int passesRunBeforeLateChunk = debugBuild ? ReferenceChainTestHooks.referenceChainPassesRunForTest0() : -1; // lateChunk is deliberately NOT held in a local: the main thread's stack is a root-walk root, // so a live local lets the root enumeration admit (and, once the canary candidate aliases to // byte[] and a GC runs tagLeakInstances(), leak-tag) lateChunk directly - the resulting @@ -285,13 +284,13 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // uncapped past SEED_EPOCHS_FOR_HYSTERESIS so a wipe costing early rounds still leaves // enough rounds to requalify. hysteresisEpoch++; - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( LEAK_BUFFER_TEST_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0( + ReferenceChainTestHooks.seedTidTrendSample0( LEAK_BUFFER_TEST_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); - JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, LEAK_BUFFER.get(LATE_CHUNK_INDEX)); - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); // Only TRUST a match once passesRun() has genuinely changed since the snapshot taken right // before lateChunk was created - either advanced (the same, never-restarted search ran a @@ -306,8 +305,8 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // before the first pass, discovered instances, other representatives) is not enough. The // tag is read BEFORE dumping - the dump snapshots the same resolved-chain cache - and a // round where the current search has not resolved lateChunk's chain yet is skipped. - if (JavaProfiler.referenceChainPassesRunForTest0() != passesRunBeforeLateChunk) { - long lateChunkTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0( + if (ReferenceChainTestHooks.referenceChainPassesRunForTest0() != passesRunBeforeLateChunk) { + long lateChunkTargetTag = ReferenceChainTestHooks.getReferenceChainResolvedTargetTagForTest0( LEAK_BUFFER.get(LATE_CHUNK_INDEX)); if (lateChunkTargetTag != 0) { profiler.dump(scratchDumpPath); @@ -338,7 +337,7 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD // Same passesRun()-changed guard as the round loop. long requiredTargetTag = 0; if (debugBuild) { - if (JavaProfiler.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { + if (ReferenceChainTestHooks.referenceChainPassesRunForTest0() == passesRunBeforeLateChunk) { continue; } // The tail watches, one at a time, chunks appended to LEAK_BUFFER well after that @@ -366,10 +365,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD attemptsOnWatchedChunk = 0; } attemptsOnWatchedChunk++; - JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(LEAK_BUFFER_TEST_KLASS_ID, LEAK_BUFFER.get(watchedIndex)); - JavaProfiler.pollReferenceChainTargets0(); - requiredTargetTag = JavaProfiler.getReferenceChainResolvedTargetTagForTest0( + ReferenceChainTestHooks.pollReferenceChainTargets0(); + requiredTargetTag = ReferenceChainTestHooks.getReferenceChainResolvedTargetTagForTest0( LEAK_BUFFER.get(watchedIndex)); if (requiredTargetTag == 0) { continue; @@ -416,9 +415,9 @@ private static int[] holderStates(Field backingListField, Field elementDataField Object backing = backingListField.get(LEAK_BUFFER); Object elementData = elementDataField.get(backing); return new int[] { - JavaProfiler.getReferenceChainFrontierStateForTest0(LEAK_BUFFER), - JavaProfiler.getReferenceChainFrontierStateForTest0(backing), - JavaProfiler.getReferenceChainFrontierStateForTest0(elementData) + ReferenceChainTestHooks.getReferenceChainFrontierStateForTest0(LEAK_BUFFER), + ReferenceChainTestHooks.getReferenceChainFrontierStateForTest0(backing), + ReferenceChainTestHooks.getReferenceChainFrontierStateForTest0(elementData) }; } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java index 4aecb0b906..6c9540882b 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java @@ -19,6 +19,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_BASE; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_POOL_SIZE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,10 +40,6 @@ @Tag("slow") public class ThreadLocalLeakReferenceChainTest extends AbstractProcessProfilerTest { - // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE). - private static final long LEAK_TAG_BASE = 0x40000000L; - private static final long LEAK_TAG_POOL_SIZE = 256; - @Test void shouldCorrelateThreadLocalHeldLeakChain() throws Exception { assumeFalse(Platform.isJavaVersion(8)); @@ -68,25 +66,30 @@ void shouldCorrelateThreadLocalHeldLeakChain() throws Exception { AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("threadlocal-leak", jvmArgs, packedCommand, Collections.emptyMap(), - 150, + childTimeoutSeconds(150), line -> { if (line.startsWith(ThreadLocalLeakScenario.FOUND_MARKER) || line.equals(ThreadLocalLeakScenario.NOT_FOUND_MARKER) || line.startsWith(ThreadLocalLeakScenario.TAG_OUT_OF_POOL_MARKER) - || line.startsWith(ThreadLocalLeakScenario.NO_LIVE_OBJECT_MARKER)) { + || line.startsWith(ThreadLocalLeakScenario.NO_LIVE_OBJECT_MARKER) + || line.equals(ThreadLocalLeakScenario.HANDOFF_TIMEOUT_MARKER)) { resultLine.set(line); } if (line.startsWith("[TEST::INFO]")) { - testLogTail.addLast(line); - while (testLogTail.size() > 2000) { - testLogTail.removeFirst(); + // Synchronized: the reader thread can outlive a join timeout. + synchronized (testLogTail) { + testLogTail.addLast(line); + while (testLogTail.size() > 2000) { + testLogTail.removeFirst(); + } } } return LineConsumerResult.CONTINUE; }, null); - assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertTrue(result.inTime, "Child process did not exit within the wait timeout" + + diagnostics(testLogTail)); assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); assertNotNull(resultLine.get(), "Child process never printed a recognizable result " + "marker on stdout" + diagnostics(testLogTail)); @@ -141,25 +144,30 @@ void shouldCorrelateLeakOnThreadCreatedAfterRecordingStart() throws Exception { AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("threadlocal-leak-late", jvmArgs, packedCommand, Collections.emptyMap(), - 150, + childTimeoutSeconds(150), line -> { if (line.startsWith(ThreadLocalLeakScenario.FOUND_MARKER) || line.equals(ThreadLocalLeakScenario.NOT_FOUND_MARKER) || line.startsWith(ThreadLocalLeakScenario.TAG_OUT_OF_POOL_MARKER) - || line.startsWith(ThreadLocalLeakScenario.NO_LIVE_OBJECT_MARKER)) { + || line.startsWith(ThreadLocalLeakScenario.NO_LIVE_OBJECT_MARKER) + || line.equals(ThreadLocalLeakScenario.HANDOFF_TIMEOUT_MARKER)) { resultLine.set(line); } if (line.startsWith("[TEST::INFO]")) { - testLogTail.addLast(line); - while (testLogTail.size() > 2000) { - testLogTail.removeFirst(); + // Synchronized: the reader thread can outlive a join timeout. + synchronized (testLogTail) { + testLogTail.addLast(line); + while (testLogTail.size() > 2000) { + testLogTail.removeFirst(); + } } } return LineConsumerResult.CONTINUE; }, null); - assertTrue(result.inTime, "Child process did not exit within the wait timeout"); + assertTrue(result.inTime, "Child process did not exit within the wait timeout" + + diagnostics(testLogTail)); assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); assertNotNull(resultLine.get(), "Child process never printed a recognizable result " + "marker on stdout" + diagnostics(testLogTail)); @@ -174,7 +182,11 @@ void shouldCorrelateLeakOnThreadCreatedAfterRecordingStart() throws Exception { } /** Filtered summary of the child's TEST_LOG stream, embedded in failure messages. */ - private static String diagnostics(Deque testLogTail) { + private static String diagnostics(Deque testLogTailRef) { + List testLogTail; + synchronized (testLogTailRef) { + testLogTail = new ArrayList<>(testLogTailRef); + } if (testLogTail.isEmpty()) { return "\n(no TEST_LOG output captured - non-debug build or no reference-chain passes ran)"; } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java index 0d5c0a5ccb..a2fcf4622d 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -6,6 +6,7 @@ package com.datadoghq.profiler.referencechains; import com.datadoghq.profiler.JavaProfiler; +import com.datadoghq.profiler.ReferenceChainTestHooks; import org.openjdk.jmc.common.IMCType; import org.openjdk.jmc.common.item.IItem; import org.openjdk.jmc.common.item.IItemCollection; @@ -23,6 +24,9 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_BASE; +import static com.datadoghq.profiler.referencechains.ReferenceChainAssertions.LEAK_TAG_POOL_SIZE; + /** * The thread-local taxonomy of the leak-tag correlation use case, run inside a * separate child JVM by {@code ExternalLauncher}'s {@code threadlocal-leak} mode: @@ -80,10 +84,6 @@ private ThreadLocalLeakScenario() {} */ private static final int CHUNK_BYTES = 1_000_000; - // Mirrored from referenceChains.h (LEAK_TAG_BASE/LEAK_TAG_POOL_SIZE). - private static final long LEAK_TAG_BASE = 0x40000000L; - private static final long LEAK_TAG_POOL_SIZE = 256; - /** * Handoff for the leak thread's profiler tid and one seed chunk - the * main thread seeds LivenessTracker's per-(klass, tid) qualification with @@ -134,9 +134,17 @@ boolean awaitPublication(long timeoutMs) throws InterruptedException { /** Printed to stdout (with the tag) if no HeapLiveObject matched the chain's tag. */ public static final String NO_LIVE_OBJECT_MARKER = "[tl-correlation-no-live-object] "; + /** + * Printed to stdout if the leak thread never published its tid and seed chunk - a fixture + * failure, reported apart from a correlation miss. + */ + public static final String HANDOFF_TIMEOUT_MARKER = "[tl-correlation-handoff-timeout]"; + private static final class ScanResult { ReferenceChainAssertions.ChainMatch payloadChain; boolean payloadChainInPool; + // ReferenceChain.leakTagIncarnation of payloadChain; -1 when the field is absent. + long payloadChainIncarnation = -1; int liveObjectMatchesForChainTag; } @@ -184,7 +192,7 @@ public static void run(JavaProfiler profiler, String startCommand, for (int i = 0; i < SEED_CHUNKS; i++) { SINK.get().add(new byte[CHUNK_BYTES]); } - handoff.publish(JavaProfiler.getTid(), SINK.get().get(0)); + handoff.publish(ReferenceChainTestHooks.getTid(), SINK.get().get(0)); while (true) { try { if (rounds.poll(50, TimeUnit.MILLISECONDS) == null) { @@ -209,21 +217,21 @@ public static void run(JavaProfiler profiler, String startCommand, leakThread.start(); } if (!handoff.awaitPublication(30000)) { - System.out.println(NOT_FOUND_MARKER); + System.out.println(HANDOFF_TIMEOUT_MARKER); return; } - boolean debugBuild = "debug".equals(System.getProperty("ddprof_test.config")); + boolean debugBuild = ReferenceChainAssertions.isDebugInstrumentedBuild(); int leakTid = 0; if (debugBuild) { // Representative first, seeds second: the rep's real klass id must // own the seeds or they authorize nothing. take() drops the handoff's // reference, so the rep chunk stays reachable only via ThreadLocalMap. - JavaProfiler.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, handoff.take()); + ReferenceChainTestHooks.setKlassPopulationRepresentativeForTest0(LEAK_KLASS_ID, handoff.take()); leakTid = handoff.tid; for (int epoch = 1; epoch <= SEED_EPOCHS_FOR_HYSTERESIS; epoch++) { - JavaProfiler.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); - JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); + ReferenceChainTestHooks.seedKlassPopulationSample0(LEAK_KLASS_ID, epoch * 10, epoch); + ReferenceChainTestHooks.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, epoch * 3, epoch); } // Maintenance continues the seeded ramp (next epoch = // SEED_EPOCHS_FOR_HYSTERESIS + 1) instead of restarting at 1 - same as @@ -231,14 +239,14 @@ public static void run(JavaProfiler profiler, String startCommand, hysteresisEpoch = SEED_EPOCHS_FOR_HYSTERESIS; // Wait for the BFS thread's first full pass, then poll - same // load-bearing wait as LeakTagCorrelationScenario. - int initialPasses = JavaProfiler.referenceChainPassesRunForTest0(); + int initialPasses = ReferenceChainTestHooks.referenceChainPassesRunForTest0(); for (int i = 0; i < 300; i++) { - if (JavaProfiler.referenceChainPassesRunForTest0() != initialPasses) { + if (ReferenceChainTestHooks.referenceChainPassesRunForTest0() != initialPasses) { break; } Thread.sleep(100); } - JavaProfiler.pollReferenceChainTargets0(); + ReferenceChainTestHooks.pollReferenceChainTargets0(); } else { // Drop the handoff's chunk reference: the lambda (captured by the // Thread's runnable) retains `handoff`, so a never-taken seedChunk @@ -255,9 +263,9 @@ public static void run(JavaProfiler profiler, String startCommand, } if (debugBuild) { hysteresisEpoch++; - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + ReferenceChainTestHooks.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); } Thread.sleep(300); System.gc(); @@ -279,9 +287,9 @@ public static void run(JavaProfiler profiler, String startCommand, } if (debugBuild) { hysteresisEpoch++; - JavaProfiler.seedKlassPopulationSample0( + ReferenceChainTestHooks.seedKlassPopulationSample0( LEAK_KLASS_ID, hysteresisEpoch * 10, hysteresisEpoch); - JavaProfiler.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); + ReferenceChainTestHooks.seedTidTrendSample0(LEAK_KLASS_ID, leakTid, hysteresisEpoch * 3, hysteresisEpoch); } if (!rounds.offer(attempt)) { throw new IllegalStateException( @@ -355,6 +363,8 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { ReferenceChainAssertions.findAccessor(type, "chain"); IMemberAccessor targetTagAccessor = ReferenceChainAssertions.findAccessor(type, "targetTag"); + IMemberAccessor incarnationAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); for (IItem item : iterable) { Object chainValue = chainAccessor.getMember(item); if (!(chainValue instanceof Object[])) { @@ -387,6 +397,8 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { } result.payloadChain = new ReferenceChainAssertions.ChainMatch(chain, targetTag, 0); result.payloadChainInPool = inPool; + result.payloadChainIncarnation = incarnationAccessor != null + ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; } } } @@ -400,10 +412,16 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { ReferenceChainAssertions.findAccessor(type, "leakTag"); IMemberAccessor objectClassAccessor = ReferenceChainAssertions.findAccessor(type, "objectClass"); + IMemberAccessor incarnationAccessor = + ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); for (IItem item : iterable) { long leakTag = leakTagAccessor != null ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; - if (leakTag != result.payloadChain.targetTag) { + long incarnation = incarnationAccessor != null + ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; + // Leak tags are recycled: the tag alone is ambiguous across chunks. + if (leakTag != result.payloadChain.targetTag + || incarnation != result.payloadChainIncarnation) { continue; } Object objectClass = objectClassAccessor != null From 594023a0da3c331008e1073eabd9853ee9c7c5b0 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 11:10:48 +0200 Subject: [PATCH 15/28] Sync leak-tag spec with landed review fixes --- ...2026-10-15-leak-tag-reuse-cached-chain-identity.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md b/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md index 36a4730b8f..869dc0db14 100644 --- a/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md +++ b/docs/sphinx/specs/2026-10-15-leak-tag-reuse-cached-chain-identity.md @@ -48,7 +48,10 @@ spec. - `FrontierTable::setLeakTag` takes the stamp as a required argument with no default. - `insert()` zeroes the stamp. 3. **R-PROPAGATE (A-R4).** `buildChainEvent()` copies the entry's stamp into a new in-memory field, - `ReferenceChainEvent::_leak_tag_incarnation`. That field defaults to 0 and is never serialized. + `ReferenceChainEvent::_leak_tag_incarnation`. That field defaults to 0. It is serialized as + `ReferenceChain.leakTagIncarnation`, and `flush_table()` writes the owner's stamp as + `HeapLiveObject.leakTagIncarnation`, so consumers join on `(tag, leakTagIncarnation)`: the tag + alone is only unambiguous within one chunk. 4. **R-OWNER (A-R5).** A cached chain describes the current owner of its leak tag if and only if both of these hold: - `isLeakTag(_target_tag)`; @@ -62,8 +65,8 @@ spec. - The frontier-tag path, the `<= 0` guard and the lock order are unchanged. 6. **R-EMIT (A-R7).** `drainPendingChainEvents()` does not emit a cached chain that has a leak-tag target and fails R-OWNER. - - It also does not erase that chain. The existing eviction policy still decides when it leaves - the cache. + - It erases that chain under `_resolved_chains_lock`: stamps only advance, so the mismatch is + permanent. `cacheResolvedChain()` also refuses to cache such a chain. - Every other chain is emitted exactly as before. 7. **R-JOIN (B-R2/B-R3).** On both exit paths of `launch()` (in-time exit, and timeout followed by forced destroy), the method joins `stdoutReader` and `stderrReader` before it returns. @@ -150,7 +153,7 @@ child-process test goes through that one `launch()`. - (a) a chain whose stamp equals the current stamp is returned by the seam for `L` and emitted by `drain`; - (b) after `acquire` + `release` (+ re-`acquire`) of `L`, the same cached chain is rejected by - the seam (0) and is absent from `drain`, while `resolvedChainCount()` is unchanged (no erase); + the seam (0) and is absent from `drain`, and the drain evicts it (`resolvedChainCount()` drops); - (c) the seam still matches across a search-generation change when the stamp is unchanged; - (d) chains with a frontier-tag target are emitted regardless of pool state. From c6e6e55eda78e53189a1d2b8c5108d4ce0e6a4a2 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 11:10:49 +0200 Subject: [PATCH 16/28] Wire reference-chain JFR roundtrip path through Gradle Both the referenceChainJfrRoundtrip_ut gtest and the ddprof-test ReferenceChainJfrParserTest get the shared .jfr path from DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR; the producer deletes the file before running so the consumer never parses a stale recording. --- ddprof-lib/build.gradle.kts | 20 ++++++++++++++++++++ ddprof-test/build.gradle.kts | 16 ++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts index ac522fd238..0ce4d08667 100644 --- a/ddprof-lib/build.gradle.kts +++ b/ddprof-lib/build.gradle.kts @@ -72,6 +72,26 @@ gtest { failFast.set(true) } +// referenceChainJfrRoundtrip_ut writes a .jfr that :ddprof-test's ReferenceChainJfrParserTest +// reads back. Both sides get the path from this env var (set here and in ddprof-test's +// build.gradle.kts), and the producer deletes the file before it runs, so the consumer never +// parses a recording left behind by an older binary. +val referenceChainRoundtripJfr = + rootProject.layout.buildDirectory + .file("tmp/reference-chain-roundtrip/datadog_reference_chain_roundtrip.jfr") + .get() + .asFile +tasks.withType().matching { + it.name.startsWith("gtest") && it.name.endsWith("_referenceChainJfrRoundtrip_ut") +}.configureEach { + val jfr = referenceChainRoundtripJfr + environment("DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR", jfr.absolutePath) + doFirst { + jfr.parentFile.mkdirs() + jfr.delete() + } +} + java { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 diff --git a/ddprof-test/build.gradle.kts b/ddprof-test/build.gradle.kts index cbbb573b57..72b7832c17 100644 --- a/ddprof-test/build.gradle.kts +++ b/ddprof-test/build.gradle.kts @@ -63,9 +63,25 @@ dependencies { // Additional test task configuration beyond what the plugin provides // The plugin creates Test tasks on glibc/macOS and Exec tasks on musl // Both need the native test library to be built first +val referenceChainRoundtripJfr = + rootProject.layout.buildDirectory + .file("tmp/reference-chain-roundtrip/datadog_reference_chain_roundtrip.jfr") + .get() + .asFile +val skipGtest = project.hasProperty("skip-gtest") tasks.matching { it.name.startsWith("test") && it.name != "test" }.configureEach { // Ensure native test library is built before running tests dependsOn(":ddprof-test-native:linkLib") + + // Written by :ddprof-lib's referenceChainJfrRoundtrip_ut gtest (same path, see + // ddprof-lib/build.gradle.kts) and read by ReferenceChainJfrParserTest. + val jfr = referenceChainRoundtripJfr + (this as? ProcessForkOptions)?.environment("DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR", jfr.absolutePath) + if (skipGtest) { + // The producing gtest does not run in this invocation: drop any file from an earlier one + // so the parser test skips instead of checking a stale recording. + doFirst { jfr.delete() } + } } // Disable the default 'test' task - we use config-specific tasks instead From 981568fcde906a2b6ccdf22d7a5e15cfa3f659fd Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 12:48:48 +0200 Subject: [PATCH 17/28] Scope reference-chain roundtrip JFR per build configuration The shared path let a multi-configuration invocation delete or overwrite another configuration's recording; both producer and consumer now derive the configuration from the task name. Also fail the build when no gtest*_referenceChainJfrRoundtrip_ut task matches, instead of the parser test silently degrading to a skip. --- ddprof-lib/build.gradle.kts | 30 +++++++++++++++++++++--------- ddprof-test/build.gradle.kts | 19 ++++++++++--------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts index 0ce4d08667..ff194cccf6 100644 --- a/ddprof-lib/build.gradle.kts +++ b/ddprof-lib/build.gradle.kts @@ -74,17 +74,29 @@ gtest { // referenceChainJfrRoundtrip_ut writes a .jfr that :ddprof-test's ReferenceChainJfrParserTest // reads back. Both sides get the path from this env var (set here and in ddprof-test's -// build.gradle.kts), and the producer deletes the file before it runs, so the consumer never -// parses a recording left behind by an older binary. -val referenceChainRoundtripJfr = - rootProject.layout.buildDirectory - .file("tmp/reference-chain-roundtrip/datadog_reference_chain_roundtrip.jfr") +// build.gradle.kts). The path is scoped per build configuration (derived from the gtest task +// name, which encodes it) so a multi-configuration invocation cannot delete or overwrite +// another configuration's recording, and the producer deletes the file before it runs, so +// the consumer never parses a recording left behind by an older binary. +val roundtripSuiteSuffix = "_referenceChainJfrRoundtrip_ut" +val roundtripProducers = tasks.withType().matching { + it.name.startsWith("gtest") && it.name.endsWith(roundtripSuiteSuffix) +} +// The gtest plugin registers one producer per build configuration; zero matches can only +// mean the suite id or the gtest task naming changed. Fail the build instead of letting +// ReferenceChainJfrParserTest silently degrade to a skip. +gradle.projectsEvaluated { + check(roundtripProducers.isNotEmpty()) { + "No gtest task matching 'gtest*$roundtripSuiteSuffix' was registered - the " + + "reference-chain JFR roundtrip wiring is stale (suite renamed?)" + } +} +roundtripProducers.configureEach { + val config = name.removePrefix("gtest").removeSuffix(roundtripSuiteSuffix).lowercase() + val jfr = rootProject.layout.buildDirectory + .file("tmp/reference-chain-roundtrip/$config/datadog_reference_chain_roundtrip.jfr") .get() .asFile -tasks.withType().matching { - it.name.startsWith("gtest") && it.name.endsWith("_referenceChainJfrRoundtrip_ut") -}.configureEach { - val jfr = referenceChainRoundtripJfr environment("DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR", jfr.absolutePath) doFirst { jfr.parentFile.mkdirs() diff --git a/ddprof-test/build.gradle.kts b/ddprof-test/build.gradle.kts index 72b7832c17..e0fa1e7a23 100644 --- a/ddprof-test/build.gradle.kts +++ b/ddprof-test/build.gradle.kts @@ -63,23 +63,24 @@ dependencies { // Additional test task configuration beyond what the plugin provides // The plugin creates Test tasks on glibc/macOS and Exec tasks on musl // Both need the native test library to be built first -val referenceChainRoundtripJfr = - rootProject.layout.buildDirectory - .file("tmp/reference-chain-roundtrip/datadog_reference_chain_roundtrip.jfr") - .get() - .asFile val skipGtest = project.hasProperty("skip-gtest") tasks.matching { it.name.startsWith("test") && it.name != "test" }.configureEach { // Ensure native test library is built before running tests dependsOn(":ddprof-test-native:linkLib") - // Written by :ddprof-lib's referenceChainJfrRoundtrip_ut gtest (same path, see - // ddprof-lib/build.gradle.kts) and read by ReferenceChainJfrParserTest. - val jfr = referenceChainRoundtripJfr + // Read back the .jfr written by :ddprof-lib's referenceChainJfrRoundtrip_ut gtest for the + // same configuration (the task name encodes it; the path is scoped per configuration in + // ddprof-lib/build.gradle.kts, which see). The profiler-test plugin's test -> + // gtest wiring already orders that producer ahead of this task. + val config = name.removePrefix("test").removePrefix("Slow").lowercase() + val jfr = rootProject.layout.buildDirectory + .file("tmp/reference-chain-roundtrip/$config/datadog_reference_chain_roundtrip.jfr") + .get() + .asFile (this as? ProcessForkOptions)?.environment("DDPROF_REFERENCE_CHAIN_ROUNDTRIP_JFR", jfr.absolutePath) if (skipGtest) { // The producing gtest does not run in this invocation: drop any file from an earlier one - // so the parser test skips instead of checking a stale recording. + // so the parser test skips (Assumptions gate) instead of checking a stale recording. doFirst { jfr.delete() } } } From e4349598bdb16d6c04d8966f8a46b751389e6ac5 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 14:47:50 +0200 Subject: [PATCH 18/28] Buffer the PersistentAllocator completion signal A SynchronousQueue drops the worker's bare offer() when the batch finishes between the handoff rendezvous and the caller entering poll(), turning a completed batch (or its rethrown failure) into a 30-second poll timeout. A one-slot ArrayBlockingQueue retains it; the protocol is single-in-flight so one slot suffices. --- .../profiler/referencechains/PersistentAllocator.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java index 5147ec909e..52c28ee8c0 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java @@ -7,6 +7,7 @@ import com.datadoghq.profiler.ReferenceChainTestHooks; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; @@ -30,8 +31,13 @@ final class PersistentAllocator implements AutoCloseable { private static final long HANDOFF_TIMEOUT_SECONDS = 30; private final SynchronousQueue handoff = new SynchronousQueue<>(); - // Worker-to-caller completion slot: null = success, Throwable = the batch's failure. - private final SynchronousQueue completion = new SynchronousQueue<>(); + // Worker-to-caller completion slot: an Object marker = success, a Throwable = the batch's + // failure. It is buffered (one slot) because the protocol is single-in-flight, and the + // worker's offer() must retain the signal even when the batch finishes in the window + // between the handoff rendezvous and the caller entering poll(): a SynchronousQueue would + // drop that bare offer and turn a completed batch (or its rethrown failure) into a + // 30-second poll timeout. + private final ArrayBlockingQueue completion = new ArrayBlockingQueue<>(1); private final Thread thread; private volatile int tid; private volatile boolean closed; From c06c96bb667e98d088ea82b528eafc21e7f2e48d Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 17:23:25 +0200 Subject: [PATCH 19/28] Fix atomic stamp read and add heap-floor reset seam bumpLeakTagIncarnationLocked() read _leak_tag_incarnation with a plain read while lock-free acquire readers race it; use an atomic load for the read half of the release-store bump. Add a heapFloorResetForTest() seam (empty head/fill, storeRelease like the producer) with JNI wiring, so the AggressiveLeak tests can clear the seeded ring instead of leaving ~991MB samples for later tests in the shared JVM. Correct getTid()'s javadoc: getTid0() is compiled unconditionally (outside javaApi.cpp's #ifdef DEBUG seam block) and works in release builds too. --- ddprof-lib/src/main/cpp/javaApi.cpp | 13 +++++++++++ ddprof-lib/src/main/cpp/livenessTracker.h | 22 +++++++++++++++++-- .../com/datadoghq/profiler/JavaProfiler.java | 18 ++++++++++++--- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index a38fe15941..e512b8acd1 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1396,6 +1396,15 @@ Java_com_datadoghq_profiler_JavaProfiler_setHeapFloorRecordingForTest0( LivenessTracker::instance()->setHeapFloorRecordingForTest(enabled == JNI_TRUE); } +// Clears the heap-floor rings seeded by heapFloorRecordForTest0() so the +// next test in the same JVM starts from an empty ring. +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_heapFloorResetForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + LivenessTracker::instance()->heapFloorResetForTest(); +} + // Evaluates the real scheduling gate without executing a BFS pass. This is not read-only: under // the engine lock it may account completed-search pain and restart/reset terminal search state. extern "C" DLLEXPORT jboolean JNICALL @@ -1510,6 +1519,10 @@ extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_setHeapFloorRecordingForTest0( JNIEnv *env, jclass unused, jboolean enabled) {} +extern "C" DLLEXPORT void JNICALL +Java_com_datadoghq_profiler_JavaProfiler_heapFloorResetForTest0( + JNIEnv *env, jclass unused) {} + extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JavaProfiler_shouldRunPassForTest0(JNIEnv *env, jclass unused) { diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index de69a8f9e1..eb5d55b7e6 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -594,10 +594,14 @@ class alignas(alignof(SpinLock)) LivenessTracker { u32 _leak_tag_incarnation[LEAK_TAG_POOL_SIZE]; // Advances slot `idx`'s ownership-period stamp. Caller holds - // _leak_tag_pool_lock. + // _leak_tag_pool_lock. The read is atomic as well: heap callbacks and + // reference-chain code read the same slot lock-free (leakTagIncarnation()), + // and a plain read racing those loads is UB under the C++ memory model even + // though the store itself is a release store. void bumpLeakTagIncarnationLocked(int idx) { __atomic_store_n(&_leak_tag_incarnation[idx], - _leak_tag_incarnation[idx] + 1, __ATOMIC_RELEASE); + __atomic_load_n(&_leak_tag_incarnation[idx], __ATOMIC_RELAXED) + 1, + __ATOMIC_RELEASE); } jlong acquireLeakTag(); @@ -1561,6 +1565,20 @@ class alignas(alignof(SpinLock)) LivenessTracker { recordHeapFloorSampleUnchecked(used, timestamp_ns, container_used); } bool heapFloorRisingForTest() const { return heapFloorRising(); } + // Ring-only companion to the heapFloorRecordForTest() seam: clears + // _heap_floor_ring/_heap_floor_time_ring/_container_mem_ring via their + // shared head/fill - all three become empty once fill is 0 (see + // ringWindowStats()'s min-fill gate in heapFloorRising()). Seeding via + // heapFloorRecordForTest() leaves "used" values behind that the next + // test in the same JVM would otherwise inherit into its secondsToOOM() + // projection. storeRelease() matches recordHeapFloorSampleUnchecked()'s + // own publication of these indices; call while heap-floor recording is + // disabled (the state every seeding test already holds) so a concurrent + // real recordHeapFloorSample() cannot interleave with the reset. + void heapFloorResetForTest() { + storeRelease(_heap_floor_ring_head, (u8)0); + storeRelease(_heap_floor_ring_fill, (u8)0); + } // Bypasses initialize_table()'s JNI-dependent HeapUsage::getMaxHeap() call // (out of gtest's reach, same reason setGcGenerationsForTest() exists) so // secondsToOOM() can be exercised directly against a fake max heap size. diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index ae90f59124..1b1a7393f8 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -463,10 +463,12 @@ public Map getDebugCounters() { private static native int getTid0(); /** - * Test seam (debug native builds only): returns the calling thread's profiler tid + * Test seam: returns the calling thread's profiler tid * (ProfiledThread::currentTid()), the tid space {@code seedTidTrendSample0} matches - * against. Must be called ON the leaking thread, or the seeded tid will match no - * tracked instance and no leak tag will be assigned. + * against. Unlike the tracker-mutating seams it only reads thread state, so it is + * compiled unconditionally and works in release builds too. Must be called ON the + * leaking thread, or the seeded tid will match no tracked instance and no leak tag + * will be assigned. */ static int getTid() { return getTid0(); @@ -703,6 +705,16 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l */ static native void setHeapFloorRecordingForTest0(boolean enabled); + /** + * Test seam (debug native builds only): clears the heap-floor ring seeded via + * {@link #heapFloorRecordForTest0(long, long)} so the next test in the same JVM does not + * inherit this test's samples into its {@code secondsToOOM()} projection. Must be called + * while heap-floor recording is disabled (i.e. before the + * {@link #setHeapFloorRecordingForTest0(boolean)} restore in a finally block) so a real GC + * cannot interleave a sample with the reset. + */ + static native void heapFloorResetForTest0(); + /** * Test seam (debug native builds only): reports whether ReferenceChainTracker's search-restart * gate ({@code canAffordNewSearch()} -> {@code hasLeakSignal()}) would currently allow a From 27f49a7d0a76b9352d133aac2fe3829ca67a6a62 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 17:23:31 +0200 Subject: [PATCH 20/28] Harden leak-identity tests: stamp guard, stale search, table clear Expose cacheResolvedChain()'s verdict through the test accessor and assert dead-owner chains are refused (also after the tag is recycled for a new owner with the dead owner's stamp). Add FrontierChainFromSupersededSearchIsRejected: a frontier chain cached by an earlier search must not answer the seam after a restart bumps _search_start_ns, and nothing answers with no active search. Add GenerationChangeClearsCachedKlassIdsWholeTable: every row below _table_size clears its cached class id on a class-map generation change and a canary row at _table_size stays untouched, pinning the loop bound. --- .../src/test/cpp/livenessTracker_ut.cpp | 43 +++++++++++++++++++ .../test/cpp/referenceChainsTestAccessors.h | 6 ++- .../test/cpp/referenceChainsTrackerTests.inc | 34 ++++++++++++++- 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp index 0ec21c4549..5b6e10f90f 100644 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp @@ -1126,6 +1126,17 @@ class LivenessTrackerTestAccessor { t->_table_max_cap = table_max_cap; } + // Writes a canary cached class id into a table row (possibly one past + // _table_size, still within _table_cap) so tests can pin whole-table + // clear-loop bounds. Returns false when idx is outside the allocation. + static bool setCachedKlassIdForTest(LivenessTracker *t, u32 idx, u32 id) { + if (t->_table == nullptr || idx >= (u32)t->_table_cap) { + return false; + } + t->_table[idx].cached_klass_id = id; + return true; + } + static int tableSizeFieldForTest(LivenessTracker *t) { return t->_table_size; } @@ -2746,6 +2757,38 @@ TEST_F(LivenessTrackerMockJvmTest, StartRebuildBumpsStampOnlyWhenOwnershipFlips) << "a tag that stays free keeps its stamp"; } +// classMapGenerationSetForTest()'s whole-table cached_klass_id clear: every row +// below _table_size loses its cached id (those ids resolved against the old +// class-map namespace and could alias an unrelated class), and the row at +// _table_size - a canary inside the allocated table but outside the valid +// rows - keeps its value, pinning the loop bound (<, not <=). +TEST_F(LivenessTrackerMockJvmTest, GenerationChangeClearsCachedKlassIdsWholeTable) { + LivenessTracker *tracker = LivenessTracker::instance(); + trackNew(&jvm.retained_class, /*tid=*/7); + trackNew(&jvm.retained_class, /*tid=*/7); + sweep(); + ASSERT_EQ(2u, tracker->tableSizeForTest()); + ASSERT_NE(0u, cachedKlassIdAt(0)); + ASSERT_NE(0u, cachedKlassIdAt(1)); + + constexpr u32 kCanaryId = 0xC0FFEE; + // Row 2 is one past the valid rows but inside the allocated table (the + // canary write fails the test if the allocation is exactly 2 rows). + ASSERT_TRUE(LivenessTrackerTestAccessor::setCachedKlassIdForTest( + tracker, /*idx=*/2, kCanaryId)); + + // Two calls make the generation change deterministic even if a previous + // test already adopted generation + 1: first aligns, second changes. + u64 generation = Profiler::instance()->classMap()->generation(); + tracker->classMapGenerationSetForTest(generation); + tracker->classMapGenerationSetForTest(generation + 1); + + EXPECT_EQ(0u, cachedKlassIdAt(0)) << "row 0's cached id must clear on a generation change"; + EXPECT_EQ(0u, cachedKlassIdAt(1)) << "row 1's cached id must clear on a generation change"; + EXPECT_EQ(kCanaryId, cachedKlassIdAt(2)) + << "the clear loop must stop at _table_size, not write the row past it"; +} + // Profiler::start() resets the class map, restarting its ids at 1, while the // tracking table and its cached class ids survive. A flush before the next GC // must resolve the class again instead of emitting the cached id, which now diff --git a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h index 2aaad343bd..a7045e8582 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h +++ b/ddprof-lib/src/test/cpp/referenceChainsTestAccessors.h @@ -221,9 +221,11 @@ class ReferenceChainsTestAccessor { ReferenceChainTracker::instance()->drainPendingChainEvents(out); } - static void cacheChain(jlong source_tag, ReferenceChainEvent event, + // Returns cacheResolvedChain()'s own verdict so tests can assert a + // dead-owner chain is refused, not merely absent from the snapshot. + static bool cacheChain(jlong source_tag, ReferenceChainEvent event, jlong source_tag_val, u64 source_search_ns) { - ReferenceChainTracker::instance()->cacheResolvedChain( + return ReferenceChainTracker::instance()->cacheResolvedChain( source_tag, std::move(event), source_tag_val, source_search_ns); } diff --git a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc index 096af1e805..44f68b6e8b 100644 --- a/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc +++ b/ddprof-lib/src/test/cpp/referenceChainsTrackerTests.inc @@ -898,7 +898,9 @@ TEST_F(ResolvedChainCacheTest, LeakTagChainOfDeadOwnerIsNeitherResolvedNorEmitte } // A chain built from a binding whose leak-tag owner already died is refused by the cache: it -// would only occupy a MAX_RESOLVED_CHAINS slot until the next dump evicts it. +// would only occupy a MAX_RESOLVED_CHAINS slot until the next dump evicts it. Recycling the tag +// for a new owner must not revive the dead owner's chain either - the new ownership period's +// stamp does not match the stale one the chain carries. TEST_F(ResolvedChainCacheTest, DeadOwnerChainIsNotCached) { LivenessTracker *lt = LivenessTracker::instance(); jlong leak = lt->acquireLeakTagForTest(); @@ -906,9 +908,37 @@ TEST_F(ResolvedChainCacheTest, DeadOwnerChainIsNotCached) { u32 stale_stamp = lt->leakTagIncarnation(leak); lt->releaseLeakTagForTest(leak); ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); - ReferenceChainsTestAccessor::cacheChain(5, makeLeakEvent(leak, stale_stamp), 5, 1000); + EXPECT_FALSE(ReferenceChainsTestAccessor::cacheChain( + 5, makeLeakEvent(leak, stale_stamp), 5, 1000)) + << "a chain whose leak-tag owner already died must be refused by the cache"; EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); EXPECT_FALSE(drainContainsTarget((u64)leak)); + + // The recycled tag's new owner must not inherit the dead owner's stamp. + jlong reused = lt->acquireLeakTagForTest(); + ASSERT_EQ(leak, reused); + EXPECT_FALSE(ReferenceChainsTestAccessor::cacheChain( + 5, makeLeakEvent(reused, stale_stamp), 5, 1000)) + << "a chain carrying the dead owner's stamp must be refused for the new owner too"; + EXPECT_EQ(0u, ReferenceChainsTestAccessor::resolvedChainCount()); +} + +// Only the current search's cached chain counts: after a restart moved _search_start_ns, a +// frontier chain cached by the superseded search must not answer the seam (the +// source_search_ns != current gate), and with no active search (start ns 0) nothing answers. +TEST_F(ResolvedChainCacheTest, FrontierChainFromSupersededSearchIsRejected) { + ReferenceChainsTestAccessor::setSearchStartNsForTest(1000); + ASSERT_TRUE(ReferenceChainsTestAccessor::cacheChain(5, makeEvent(77), 5, 1000)); + EXPECT_EQ(77, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(5)); + + // Restart: the cached chain belongs to the superseded search. + ReferenceChainsTestAccessor::setSearchStartNsForTest(2000); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(5)) + << "a frontier chain cached by an earlier search must not answer the seam"; + + // No active search: the gate stays closed even though a chain is cached. + ReferenceChainsTestAccessor::setSearchStartNsForTest(0); + EXPECT_EQ(0, ReferenceChainTracker::instance()->resolvedChainTargetTagForTest(5)); } // Within one ownership period the leak-tag path is generation-agnostic: a chain an earlier search From c938f5c2e16f48794fca2f025ec6e92182869560 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 17:23:36 +0200 Subject: [PATCH 21/28] Fix leak-correlation evidence handling in reference-chain scenarios PersistentAllocator.run() clears the completion slot before the handoff so a stale outcome from a timed-out batch can never be attributed to the next task (the worker is parked at handoff.take() at that point). The AggressiveLeak tests reset the heap-floor ring in finally before re-enabling recording. ThreadLocalLeakScenario: keepBest keeps an in-pool chain over a later out-of-pool one, and an unresolved -1 incarnation fails loud or is a hard mismatch instead of degrading to tag-only correlation. LeakTagCorrelationScenario: keepBest ranks scans by the same evidence score scan() uses (owner-verified 2, in-pool 1), so later degraded scans cannot displace the best correlation. --- .../profiler/ReferenceChainTestHooks.java | 10 ++++ .../AggressiveLeakReferenceChainTest.java | 6 +++ .../LeakTagCorrelationScenario.java | 20 +++++++- .../referencechains/PersistentAllocator.java | 6 +++ .../ThreadLocalLeakScenario.java | 47 ++++++++++++++++--- 5 files changed, 81 insertions(+), 8 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java index a070e32462..3701f531ae 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java @@ -91,6 +91,16 @@ public static void setHeapFloorRecordingForTest0(boolean enabled) { JavaProfiler.setHeapFloorRecordingForTest0(enabled); } + /** + * Clears the heap-floor ring seeded via {@link #heapFloorRecordForTest0(long, long)} so + * the next test in the same JVM starts from an empty ring. Call before the + * {@link #setHeapFloorRecordingForTest0(boolean)} restore, while recording is still + * disabled, so a real GC cannot interleave a sample with the reset. + */ + public static void heapFloorResetForTest0() { + JavaProfiler.heapFloorResetForTest0(); + } + /** * Not a read-only query: under the native engine lock the gate may account a completed * search's pain and restart/reset terminal search state. diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java index 7552854696..e900355491 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java @@ -120,6 +120,10 @@ public void shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate() "The passes that just ran must still have had zero per-klass candidates - the " + "aggregate urgent-OOM projection was the only possible trigger"); } finally { + // Reset the ring while recording is still disabled so the restore below cannot + // let a real GC interleave a sample with the reset, and so the next test in this + // JVM does not inherit this test's seeded samples into its secondsToOOM(). + ReferenceChainTestHooks.heapFloorResetForTest0(); ReferenceChainTestHooks.setMaxHeapBytesForTest0(-1); ReferenceChainTestHooks.setHeapFloorRecordingForTest0(true); } @@ -151,6 +155,8 @@ public void shouldNotOpenSearchGateOnFlatHeapFloorWithNoLeakCandidate() { "Expected the search-restart gate to stay closed: no heap growth and no per-klass " + "candidate give hasLeakSignal() nothing to trust"); } finally { + // Same order as the seeding test above: ring reset first, still disabled. + ReferenceChainTestHooks.heapFloorResetForTest0(); ReferenceChainTestHooks.setMaxHeapBytesForTest0(-1); ReferenceChainTestHooks.setHeapFloorRecordingForTest0(true); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index 7638c3d387..961e9a272c 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -526,7 +526,18 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { if (scan == null) { return best; } - ScanResult result = scan.payloadChain != null || best == null ? scan : best; + if (best == null) { + return scan; + } + // Same evidence ranking scan() applies within one dump: an owner-verified + // in-pool chain (2) outranks an unverified in-pool one (1), which outranks + // any out-of-pool chain (0). A later scan only wins on equal or better + // evidence, so noise churn and tag recycling cannot displace the + // best-evidenced correlation mid-run. + int bestScore = evidenceScore(best); + int scanScore = evidenceScore(scan); + ScanResult result = scanScore > bestScore || (scanScore == bestScore + && scan.payloadChain != null) ? scan : best; ScanResult other = result == scan ? best : scan; if (other != null) { mergeDistinct(result.transientViolations, other.transientViolations); @@ -535,6 +546,13 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { return result; } + private static int evidenceScore(ScanResult scan) { + if (scan.payloadChain == null) { + return -1; + } + return scan.payloadChainInPool ? (scan.payloadChainOwnerVerified ? 2 : 1) : 0; + } + private static void mergeDistinct(List into, List from) { for (String s : from) { if (!into.contains(s)) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java index 52c28ee8c0..30cced407d 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java @@ -76,6 +76,12 @@ void run(Runnable task) { throw new IllegalStateException("allocator thread " + thread.getName() + " is no longer running"); } try { + // A previous run() that timed out left its outcome in the slot (the worker + // finishes the abandoned task and offers into the then-empty queue). Drop it + // before the handoff so it can never be attributed to this task: the worker is + // parked at handoff.take() at this point, so nothing new can arrive between + // the clear and the rendezvous. + completion.clear(); if (!handoff.offer(task, HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { throw new IllegalStateException("allocator thread " + thread.getName() + " did not accept a task"); } diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java index a2fcf4622d..1fe37317f8 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -332,10 +332,19 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { if (scan == null) { return best; } - if (scan.payloadChain != null) { + if (best == null || best.payloadChain == null) { return scan; } - return best != null ? best : scan; + if (scan.payloadChain == null) { + return best; + } + // Mirror scan()'s own within-scan preference: an in-pool (correlated) + // chain outranks an out-of-pool one no matter which scan observed it, so + // a later out-of-pool chain cannot displace an earlier correlated one. + if (best.payloadChainInPool != scan.payloadChainInPool) { + return best.payloadChainInPool ? best : scan; + } + return scan; } /** @@ -397,8 +406,21 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { } result.payloadChain = new ReferenceChainAssertions.ChainMatch(chain, targetTag, 0); result.payloadChainInPool = inPool; - result.payloadChainIncarnation = incarnationAccessor != null - ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; + // The (tag, incarnation) pair is the chain's identity; an unresolved + // -1 stamp must fail loud rather than degrade to tag-only matching, + // which a recycled tag's chunk could satisfy. + if (incarnationAccessor == null) { + throw new IllegalStateException( + "datadog.ReferenceChain events have no leakTagIncarnation accessor"); + } + long chainIncarnation = + ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)); + if (chainIncarnation == -1) { + throw new IllegalStateException( + "chain leakTagIncarnation is not a numeric value; " + + "tag-only correlation is ambiguous across recycled tags"); + } + result.payloadChainIncarnation = chainIncarnation; } } } @@ -414,13 +436,24 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { ReferenceChainAssertions.findAccessor(type, "objectClass"); IMemberAccessor incarnationAccessor = ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); + // Fail loud if the field is missing: without it every live object + // reads -1 and the correlation below would degrade to tag-only + // matching, which a recycled tag's stale chunk could satisfy. + if (incarnationAccessor == null) { + throw new IllegalStateException( + "datadog.HeapLiveObject events have no leakTagIncarnation accessor"); + } for (IItem item : iterable) { long leakTag = leakTagAccessor != null ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; - long incarnation = incarnationAccessor != null - ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; + long incarnation = + ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)); // Leak tags are recycled: the tag alone is ambiguous across chunks. - if (leakTag != result.payloadChain.targetTag + // An unresolved -1 incarnation is a hard mismatch, never a tag-only + // match: a stale chunk that inherited the recycled tag must not + // count as this chain's live object. + if (incarnation == -1 + || leakTag != result.payloadChain.targetTag || incarnation != result.payloadChainIncarnation) { continue; } From acd0f0485d137188ec923ac353165c337b6fd8eb Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 17:23:47 +0200 Subject: [PATCH 22/28] Share roundtrip constants and fail loud on unset ddprof_test.config The roundtrip gtest writes its seeded values (class names, edge labels, target tag, depth, incarnation) to a .expected sidecar next to the JFR and the Java parser test asserts against it, removing the duplicated hard-coded expectations across the two modules. Add ReferenceChainAssertions.requiredTestConfig(): child-JVM launchers fail loud when ddprof_test.config is unset instead of forwarding the literal null that the child would classify as a release build. ExternalProcessReferenceChainTest derives its expected seam path from isDebugInstrumentedBuild(), the same single encoding the child uses. --- .../cpp/referenceChainJfrRoundtrip_ut.cpp | 35 ++++++++++-- .../ExternalProcessReferenceChainTest.java | 12 +++-- .../LeakTagCorrelationReferenceChainTest.java | 2 +- .../ReferenceChainAssertions.java | 16 ++++++ .../ReferenceChainJfrParserTest.java | 53 ++++++++++++++----- .../ThreadLocalLeakReferenceChainTest.java | 4 +- 6 files changed, 98 insertions(+), 24 deletions(-) diff --git a/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp b/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp index 74ff19d803..b69987128f 100644 --- a/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp +++ b/ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp @@ -321,12 +321,18 @@ TEST_F(ReferenceChainJfrRoundtripTest, ProducesValidStandaloneJfrWithChainEvent) // serializes into the T_CLASS checkpoint - this is the exact API // Profiler::lookupClass() already-existing tests use (e.g. // referenceChains_ut.cpp's ReconstructsChainForSyntheticGraph). + // The constants feed both the lookup and the .expected sidecar below, so + // the companion Java parser test reads the seeded values from a single + // source instead of duplicating them. + static constexpr const char *kLeafClass = "com/test/ChainLeaf"; + static constexpr const char *kMiddleClass = "com/test/ChainMiddle"; + static constexpr const char *kRootClass = "com/test/ChainRoot"; int leafKlass = Profiler::instance()->lookupClass( - "com/test/ChainLeaf", strlen("com/test/ChainLeaf")); + kLeafClass, strlen(kLeafClass)); int middleKlass = Profiler::instance()->lookupClass( - "com/test/ChainMiddle", strlen("com/test/ChainMiddle")); + kMiddleClass, strlen(kMiddleClass)); int rootKlass = Profiler::instance()->lookupClass( - "com/test/ChainRoot", strlen("com/test/ChainRoot")); + kRootClass, strlen(kRootClass)); ASSERT_NE(-1, leafKlass); ASSERT_NE(-1, middleKlass); ASSERT_NE(-1, rootKlass); @@ -388,6 +394,29 @@ TEST_F(ReferenceChainJfrRoundtripTest, ProducesValidStandaloneJfrWithChainEvent) // same finalization every real recording chunk goes through. } + // The values this test seeds, written next to the recording: the companion + // Java parser test (ReferenceChainJfrParserTest) asserts against this + // sidecar instead of hard-coding them, so a change to the seeding shows + // up as a deliberate gtest-only diff rather than an opaque assertion + // failure in the other module. The Java side maps the slashed internal + // names to JMC's dotted IMCType full names. + { + std::string expectedPath = path + ".expected"; + FILE *expected = fopen(expectedPath.c_str(), "w"); + ASSERT_NE(nullptr, expected) << "could not open " << expectedPath; + fprintf(expected, "chain.length=%d\n", 3); + fprintf(expected, "chain.0=%s\n", kLeafClass); + fprintf(expected, "chain.1=%s\n", kMiddleClass); + fprintf(expected, "chain.2=%s\n", kRootClass); + fprintf(expected, "edges.0=field\n"); + fprintf(expected, "edges.1=element\n"); + fprintf(expected, "edges.2=jni_global\n"); + fprintf(expected, "targetTag=%lld\n", (long long)event._target_tag); + fprintf(expected, "depth=%u\n", event._depth); + fprintf(expected, "leakTagIncarnation=%u\n", event._leak_tag_incarnation); + fclose(expected); + } + FILE *f = fopen(path.c_str(), "rb"); ASSERT_NE(nullptr, f) << "expected " << path << " to have been written"; char magic[4] = {0, 0, 0, 0}; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java index 883594abf0..55da7b61ff 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ExternalProcessReferenceChainTest.java @@ -75,8 +75,10 @@ void shouldReconstructReferrerChainInSeparateProcess() throws Exception { // ddprof_test.config is set on the Test task by ProfilerTestPlugin.kt and is not inherited // by a ProcessBuilder-launched child; the scenario needs it for its debug-only seeding. + // requiredTestConfig() also fails loud when the property is unset instead of forwarding + // the literal "null", which the child would classify as a release build. List jvmArgs = Collections.singletonList( - "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + "-Dddprof_test.config=" + ReferenceChainAssertions.requiredTestConfig()); AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("leak-cache", jvmArgs, packedCommand, @@ -147,7 +149,7 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); List jvmArgs = new ArrayList<>(); - jvmArgs.add("-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + jvmArgs.add("-Dddprof_test.config=" + ReferenceChainAssertions.requiredTestConfig()); // --add-opens: the scenario reflects into Collections$SynchronizedList.list and // ArrayList.elementData to check its holders' expansion state before creating lateChunk. // JDK 8 has no module system (that reflection needs no opening) and rejects the option. @@ -178,9 +180,9 @@ void shouldReconstructReferrerChainForGrowingStaticFieldCollection() throws Exce assertTrue(result.inTime, "Child process did not exit within the wait timeout"); assertEquals(0, result.exitCode, "Child process exited with a non-zero code"); // The scenario's native test seams are compiled into every -DDEBUG build (debug/asan/tsan), - // so the strong seam path must be taken on exactly those configs. - String config = System.getProperty("ddprof_test.config"); - boolean expectSeams = config != null && ("debug".equals(config) || config.endsWith("san")); + // so the strong seam path must be taken on exactly those configs - the same single + // encoding the child itself uses, not an independent string match here. + boolean expectSeams = ReferenceChainAssertions.isDebugInstrumentedBuild(); assertEquals(StaticFieldGrowingCollectionScenario.SEAM_PATH_MARKER + expectSeams, seamPathLine.get(), "scenario's seam-path predicate must match the native -DDEBUG build set"); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java index b0455e247e..d292f78753 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationReferenceChainTest.java @@ -127,7 +127,7 @@ private void assertCorrelationFound(String pauseTargetOption, String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); List jvmArgs = Collections.singletonList( - "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + "-Dddprof_test.config=" + ReferenceChainAssertions.requiredTestConfig()); AtomicReference resultLine = new AtomicReference<>(); // Level 2 is required for the FRONTIER-selection TEST_LOG line (rcDebugLevel.h); the diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java index dce3a2d76b..735720170f 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainAssertions.java @@ -51,6 +51,22 @@ public static boolean isDebugInstrumentedBuild() { return config != null && ("debug".equals(config) || config.endsWith("san")); } + /** + * The {@code ddprof_test.config} value ProfilerTestPlugin.kt sets on every test<config> task. + * Launchers forwarding it to a child JVM must fail loud when it is unset (a test run directly + * from an IDE, outside the Gradle tasks) instead of concatenating the literal "null", which the + * child would silently classify as a release build and take the unseeded, flakier path. + */ + public static String requiredTestConfig() { + String config = System.getProperty("ddprof_test.config"); + if (config == null) { + throw new IllegalStateException( + "ddprof_test.config is not set; run via a test Gradle task so the child " + + "JVM sees the right build configuration"); + } + return config; + } + /** Result of {@link #findMatchForClass(IItemCollection, Class)}: one resolved chain event's fields. */ public static final class ChainMatch { public final List chain; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java index 9440885bd3..555ff34cc7 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java @@ -68,6 +68,18 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() "Expected " + jfrPath + " to exist - run " + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " + "(referenceChainJfrRoundtrip_ut.cpp produces this file)."); + // The seeded values the companion gtest wrote next to the recording - the + // single source of truth for this test's expectations, so the producer and + // this consumer cannot drift apart silently. + Path expectedPath = Paths.get(configured + ".expected"); + Assumptions.assumeTrue(Files.exists(expectedPath), + "Expected " + expectedPath + " to exist - run " + + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " + + "(referenceChainJfrRoundtrip_ut.cpp produces this sidecar)."); + java.util.Properties expected = new java.util.Properties(); + try (InputStream in = Files.newInputStream(expectedPath)) { + expected.load(in); + } IItemCollection events; try (InputStream in = Files.newInputStream(jfrPath)) { @@ -121,15 +133,16 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() "'edges' field resolved to " + edgesValue.getClass() + ", expected an array"); Object[] rawEdges = (Object[]) edgesValue; - assertEquals(3, rawEdges.length, "edges must align with the chain, one label per hop"); + assertEquals(expectedInt(expected, "chain.length"), rawEdges.length, + "edges must align with the chain, one label per hop"); for (Object e : rawEdges) { assertTrue(e instanceof String, "edges[] element resolved to " + (e == null ? "null" : e.getClass()) + ", expected a String"); } - assertEquals("field", rawEdges[0]); - assertEquals("element", rawEdges[1]); - assertEquals("jni_global", rawEdges[2]); + assertEquals(expected.getProperty("edges.0"), rawEdges[0]); + assertEquals(expected.getProperty("edges.1"), rawEdges[1]); + assertEquals(expected.getProperty("edges.2"), rawEdges[2]); if (targetTagAccessor != null) { Object v = targetTagAccessor.getMember(item); if (v instanceof Number) { @@ -155,17 +168,31 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() } assertNotNull(resolvedChain, "Never iterated a " + EVENT_TYPE + " item"); - assertEquals(3, resolvedChain.size(), "Expected leaf/middle/root - 3 entries"); + assertEquals(expectedInt(expected, "chain.length"), resolvedChain.size(), + "Expected leaf/middle/root - as many entries as the gtest seeded"); // Leaf-to-root order, matching ReferenceChainTracker::buildChainEvent()'s // FrontierTable::reconstructChain() contract and the gtest's insert() calls - // (tag=3 leaf -> tag=2 middle -> tag=1 root). - assertEquals("com.test.ChainLeaf", resolvedChain.get(0).getFullName()); - assertEquals("com.test.ChainMiddle", resolvedChain.get(1).getFullName()); - assertEquals("com.test.ChainRoot", resolvedChain.get(2).getFullName()); - assertEquals(3L, targetTag, "targetTag should be the leaf's frontier tag (3)"); - assertEquals(2, depth, "depth should be the leaf's own depth (2 hops from the root)"); - assertEquals(5L, leakTagIncarnation, - "leakTagIncarnation should carry the event's ownership stamp (5)"); + // (tag=3 leaf -> tag=2 middle -> tag=1 root). The sidecar carries the gtest's + // slashed internal names; JMC renders IMCType full names with dots. + for (int i = 0; i < resolvedChain.size(); i++) { + assertEquals(expected.getProperty("chain." + i).replace('/', '.'), + resolvedChain.get(i).getFullName(), + "chain[" + i + "] must be the gtest's seeded class in leaf-to-root order"); + } + assertEquals(expectedLong(expected, "targetTag"), targetTag, + "targetTag should be the leaf's frontier tag the gtest seeded"); + assertEquals(expectedInt(expected, "depth"), depth, + "depth should be the leaf's own depth the gtest seeded"); + assertEquals(expectedLong(expected, "leakTagIncarnation"), leakTagIncarnation, + "leakTagIncarnation should carry the event's ownership stamp the gtest seeded"); + } + + private static int expectedInt(java.util.Properties expected, String key) { + return Integer.parseInt(expected.getProperty(key)); + } + + private static long expectedLong(java.util.Properties expected, String key) { + return Long.parseLong(expected.getProperty(key)); } private static long longValue(Object v) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java index 6c9540882b..b0e631a740 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakReferenceChainTest.java @@ -61,7 +61,7 @@ void shouldCorrelateThreadLocalHeldLeakChain() throws Exception { String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); List jvmArgs = Collections.singletonList( - "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + "-Dddprof_test.config=" + ReferenceChainAssertions.requiredTestConfig()); AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("threadlocal-leak", jvmArgs, packedCommand, @@ -139,7 +139,7 @@ void shouldCorrelateLeakOnThreadCreatedAfterRecordingStart() throws Exception { String packedCommand = startCommand + "|||" + scratchDumpPath.toAbsolutePath(); List jvmArgs = Collections.singletonList( - "-Dddprof_test.config=" + System.getProperty("ddprof_test.config")); + "-Dddprof_test.config=" + ReferenceChainAssertions.requiredTestConfig()); AtomicReference resultLine = new AtomicReference<>(); LaunchResult result = launch("threadlocal-leak-late", jvmArgs, packedCommand, From 9d24c5a66a5c88ab4d23b39fa795a284b632ec8b Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 18:25:58 +0200 Subject: [PATCH 23/28] Enforce heap-floor reset precondition and assert its effect heapFloorResetForTest() published two independent storeReleases, so a concurrent reader could observe a torn old-fill/new-head window, and no store ordering closes that (seeing the first store's new value while the second is still old is possible either way). The documented precondition - heap-floor recording disabled - is what excludes every reader, so enforce it in DEBUG builds (where the flag exists): reset while recording is enabled fails an assert instead of racing silently. Expose heapFloorRisingForTest0() (JNI + hooks) and add shouldEmptyHeapFloorRingOnReset(): rising-then-empty through the real heapFloorRising() read, killing a no-op'd reset that previously survived the whole suite unobserved. --- ddprof-lib/src/main/cpp/javaApi.cpp | 15 ++++++++ ddprof-lib/src/main/cpp/livenessTracker.h | 21 +++++++++-- .../com/datadoghq/profiler/JavaProfiler.java | 15 ++++++-- .../profiler/ReferenceChainTestHooks.java | 11 +++++- .../AggressiveLeakReferenceChainTest.java | 36 +++++++++++++++++++ 5 files changed, 92 insertions(+), 6 deletions(-) diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp index e512b8acd1..f9365be145 100644 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ b/ddprof-lib/src/main/cpp/javaApi.cpp @@ -1405,6 +1405,15 @@ Java_com_datadoghq_profiler_JavaProfiler_heapFloorResetForTest0( LivenessTracker::instance()->heapFloorResetForTest(); } +// Reports whether the seeded heap-floor ring currently shows a sustained rise +// (heapFloorRising()) - the direct read over the ring the reset seam clears. +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_heapFloorRisingForTest0( + JNIEnv *env, jclass unused) { + ProfiledThread::initCurrentThreadSignalSafe(); + return LivenessTracker::instance()->heapFloorRisingForTest() ? JNI_TRUE : JNI_FALSE; +} + // Evaluates the real scheduling gate without executing a BFS pass. This is not read-only: under // the engine lock it may account completed-search pain and restart/reset terminal search state. extern "C" DLLEXPORT jboolean JNICALL @@ -1523,6 +1532,12 @@ extern "C" DLLEXPORT void JNICALL Java_com_datadoghq_profiler_JavaProfiler_heapFloorResetForTest0( JNIEnv *env, jclass unused) {} +extern "C" DLLEXPORT jboolean JNICALL +Java_com_datadoghq_profiler_JavaProfiler_heapFloorRisingForTest0( + JNIEnv *env, jclass unused) { + return JNI_FALSE; +} + extern "C" DLLEXPORT jboolean JNICALL Java_com_datadoghq_profiler_JavaProfiler_shouldRunPassForTest0(JNIEnv *env, jclass unused) { diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index eb5d55b7e6..319739ec86 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -14,6 +14,7 @@ #include "event.h" #include "spinLock.h" #include "xorshift.h" +#include #include #include #include @@ -1572,10 +1573,24 @@ class alignas(alignof(SpinLock)) LivenessTracker { // heapFloorRecordForTest() leaves "used" values behind that the next // test in the same JVM would otherwise inherit into its secondsToOOM() // projection. storeRelease() matches recordHeapFloorSampleUnchecked()'s - // own publication of these indices; call while heap-floor recording is - // disabled (the state every seeding test already holds) so a concurrent - // real recordHeapFloorSample() cannot interleave with the reset. + // own publication of these indices. The two index stores are not atomic + // as a pair - a concurrent reader could observe a torn old-fill/new-head + // window, and no store ordering closes that (seeing the first store's new + // value while the second is still old is possible either way) - so the + // documented precondition, heap-floor recording disabled, is what + // excludes every reader (onGC() skips recordHeapFloorSample() and no + // selectLeakCandidates() scan runs mid-test). Enforce it in DEBUG builds + // (where the flag exists): a caller that resets while recording is + // enabled fails the assert instead of racing a live reader. Non-DEBUG + // builds compile the reset bare - the flag does not exist there and the + // JNI entry that reaches this seam is compiled out. void heapFloorResetForTest() { +#ifdef DEBUG + if (!_heap_floor_recording_disabled_for_test.load(std::memory_order_acquire)) { + assert(!"heapFloorResetForTest() called while heap-floor recording is enabled"); + return; + } +#endif storeRelease(_heap_floor_ring_head, (u8)0); storeRelease(_heap_floor_ring_fill, (u8)0); } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java index 1b1a7393f8..ccd57efee0 100644 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java @@ -710,11 +710,22 @@ private static native void setTraceContext0(long localRootSpanId, long spanId, l * {@link #heapFloorRecordForTest0(long, long)} so the next test in the same JVM does not * inherit this test's samples into its {@code secondsToOOM()} projection. Must be called * while heap-floor recording is disabled (i.e. before the - * {@link #setHeapFloorRecordingForTest0(boolean)} restore in a finally block) so a real GC - * cannot interleave a sample with the reset. + * {@link #setHeapFloorRecordingForTest0(boolean)} restore in a finally block) - the seam + * asserts when called while recording is enabled, because a concurrent sample cannot be + * excluded from the two non-atomic index stores otherwise. */ static native void heapFloorResetForTest0(); + /** + * Test seam (debug native builds only): reports whether the seeded heap-floor ring + * currently shows a sustained rise ({@code heapFloorRising()}) - the aggregate growth + * read {@code hasLeakSignal()}'s gate uses alongside {@code secondsToOOM()}'s urgent + * projection. Returns {@code false} both before any seeding and after + * {@link #heapFloorResetForTest0()} emptied the ring, which is how tests assert the + * reset actually cleared the indices. + */ + static native boolean heapFloorRisingForTest0(); + /** * Test seam (debug native builds only): reports whether ReferenceChainTracker's search-restart * gate ({@code canAffordNewSearch()} -> {@code hasLeakSignal()}) would currently allow a diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java index 3701f531ae..c37170fa2a 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/ReferenceChainTestHooks.java @@ -95,12 +95,21 @@ public static void setHeapFloorRecordingForTest0(boolean enabled) { * Clears the heap-floor ring seeded via {@link #heapFloorRecordForTest0(long, long)} so * the next test in the same JVM starts from an empty ring. Call before the * {@link #setHeapFloorRecordingForTest0(boolean)} restore, while recording is still - * disabled, so a real GC cannot interleave a sample with the reset. + * disabled - the native seam asserts when called while recording is enabled. */ public static void heapFloorResetForTest0() { JavaProfiler.heapFloorResetForTest0(); } + /** + * Reports whether the seeded heap-floor ring currently shows a sustained rise + * ({@code heapFloorRising()}). {@code false} before any seeding and after + * {@link #heapFloorResetForTest0()} emptied the ring. + */ + public static boolean heapFloorRisingForTest0() { + return JavaProfiler.heapFloorRisingForTest0(); + } + /** * Not a read-only query: under the native engine lock the gate may account a completed * search's pain and restart/reset terminal search state. diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java index e900355491..db78767791 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/AggressiveLeakReferenceChainTest.java @@ -161,4 +161,40 @@ public void shouldNotOpenSearchGateOnFlatHeapFloorWithNoLeakCandidate() { ReferenceChainTestHooks.setHeapFloorRecordingForTest0(true); } } + + /** + * Kills a no-op'd {@code heapFloorResetForTest0()} mutation: every other call site resets in + * a finally block that no assertion observes, so without this test a reset whose body was + * dropped (or whose JNI entry was left as the release stub) would leave the whole suite + * green while leaking one test's seeded samples into the next test's JVM. The rising-then- + * empty pair is asserted directly through {@code heapFloorRising()} - the same read the + * aggregate gate uses - so the reset's effect is checked, not assumed. + */ + @Test + public void shouldEmptyHeapFloorRingOnReset() { + assumeDebugBuild(); + ReferenceChainTestHooks.setHeapFloorRecordingForTest0(false); + ReferenceChainTestHooks.resetKlassPopulationForTest0(); + ReferenceChainTestHooks.resetReferenceChainSearchForTest0(); + try { + // Same rising seeds as shouldOpenSearchGateOnAggressiveHeapWideGrowthWithNoLeakCandidate: + // 100 MiB to 991 MiB over 9 fake seconds, which clears heapFloorRising()'s growth and + // floor bars at exactly the min-fill trend window (fill = 10 samples). + for (int i = 0; i < 10; i++) { + long usedBytes = (100L + i * 99L) * 1024 * 1024; + long timestampNs = i * 1_000_000_000L; + ReferenceChainTestHooks.heapFloorRecordForTest0(usedBytes, timestampNs); + } + assertTrue(ReferenceChainTestHooks.heapFloorRisingForTest0(), + "Precondition: the seeded samples must read as a sustained rise before the reset"); + ReferenceChainTestHooks.heapFloorResetForTest0(); + assertFalse(ReferenceChainTestHooks.heapFloorRisingForTest0(), + "heapFloorResetForTest0() must empty the ring: fill = 0 falls below the min-fill " + + "trend gate heapFloorRising() requires"); + } finally { + // Same order as the other tests: ring reset first, still disabled. + ReferenceChainTestHooks.heapFloorResetForTest0(); + ReferenceChainTestHooks.setHeapFloorRecordingForTest0(true); + } + } } From f3ca357eaeff88fabfb7fb2c8dca83422b472026 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 18:26:08 +0200 Subject: [PATCH 24/28] Attribute PersistentAllocator outcomes per run, not by timing completion.clear() before the handoff was a timing-only guard: after a run() that timed out, the worker is not parked at handoff.take() yet, so the abandoned task's outcome could land after the clear and be attributed to the next task. Stamp each run with a monotonic generation carried through the Job/Outcome pair and discard stale generations at poll time; the stale entry is always offered before the current one's verdict because the worker offers it before parking at take(), through which alone the new Job is accepted. --- .../referencechains/PersistentAllocator.java | 101 +++++++++++++----- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java index 30cced407d..94441b1b43 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java @@ -30,17 +30,47 @@ final class PersistentAllocator implements AutoCloseable { private static final long HANDOFF_TIMEOUT_SECONDS = 30; - private final SynchronousQueue handoff = new SynchronousQueue<>(); - // Worker-to-caller completion slot: an Object marker = success, a Throwable = the batch's - // failure. It is buffered (one slot) because the protocol is single-in-flight, and the - // worker's offer() must retain the signal even when the batch finishes in the window - // between the handoff rendezvous and the caller entering poll(): a SynchronousQueue would - // drop that bare offer and turn a completed batch (or its rethrown failure) into a - // 30-second poll timeout. - private final ArrayBlockingQueue completion = new ArrayBlockingQueue<>(1); + // One unit of work handed to the worker; carries the caller-side generation + // so the worker can echo it back in the Outcome. The CLOSE poison rides in a + // Job too (generation -1, never a real generation). + private static final class Job { + final Runnable task; + final int generation; + + Job(Runnable task, int generation) { + this.task = task; + this.generation = generation; + } + } + + // The worker's completion entry; the generation identifies which run() + // produced it, so a stale outcome left by an abandoned (timed-out) run is + // discarded by the caller instead of being misattributed to the next task. + private static final class Outcome { + final int generation; + final Object result; // Object marker = success, a Throwable = the batch's failure + + Outcome(int generation, Object result) { + this.generation = generation; + this.result = result; + } + } + + private final SynchronousQueue handoff = new SynchronousQueue<>(); + // Worker-to-caller completion slot: an Outcome with the caller's generation + // = this task's verdict (Object marker = success, a Throwable = the batch's + // failure). It is buffered (one slot) because the protocol is single-in-flight, + // and the worker's offer() must retain the signal even when the batch finishes + // in the window between the handoff rendezvous and the caller entering poll(): + // a SynchronousQueue would drop that bare offer and turn a completed batch + // (or its rethrown failure) into a 30-second poll timeout. + private final ArrayBlockingQueue completion = new ArrayBlockingQueue<>(1); private final Thread thread; private volatile int tid; private volatile boolean closed; + // Monotonic per-run() token; only the caller thread runs tasks through run(), + // so plain int arithmetic is sufficient. + private int nextGeneration; PersistentAllocator(String name) { thread = new Thread(this::loop, name); @@ -76,24 +106,37 @@ void run(Runnable task) { throw new IllegalStateException("allocator thread " + thread.getName() + " is no longer running"); } try { - // A previous run() that timed out left its outcome in the slot (the worker - // finishes the abandoned task and offers into the then-empty queue). Drop it - // before the handoff so it can never be attributed to this task: the worker is - // parked at handoff.take() at this point, so nothing new can arrive between - // the clear and the rendezvous. + // A previous run() that timed out may still have its outcome parked in + // the slot (the worker finishes the abandoned task and offers into the + // then-empty queue). Drop what is already there; the generation check + // below then discards any stale outcome that lands after this clear - + // a plain clear alone cannot close that window, because the worker only + // parks at handoff.take() AFTER offering the abandoned task's outcome. completion.clear(); - if (!handoff.offer(task, HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + int generation = nextGeneration++; + if (!handoff.offer(new Job(task, generation), HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { throw new IllegalStateException("allocator thread " + thread.getName() + " did not accept a task"); } - Object outcome = completion.poll(HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS); - if (outcome == null) { - throw new IllegalStateException("allocator thread " + thread.getName() + " never finished a task"); - } - if (outcome instanceof RuntimeException) { - throw (RuntimeException) outcome; - } - if (outcome instanceof Error) { - throw (Error) outcome; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(HANDOFF_TIMEOUT_SECONDS); + while (true) { + Outcome outcome = completion.poll(deadline - System.nanoTime(), TimeUnit.NANOSECONDS); + if (outcome == null) { + throw new IllegalStateException("allocator thread " + thread.getName() + " never finished a task"); + } + // Discard an abandoned earlier run()'s outcome and keep waiting for this + // generation's verdict. The stale entry is always offered before our own: + // the worker offers it before looping back to handoff.take(), through + // which alone our Job can be accepted. + if (outcome.generation != generation) { + continue; + } + if (outcome.result instanceof RuntimeException) { + throw (RuntimeException) outcome.result; + } + if (outcome.result instanceof Error) { + throw (Error) outcome.result; + } + return; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -109,7 +152,7 @@ public void close() { } closed = true; try { - if (thread.isAlive() && handoff.offer(CLOSE, HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + if (thread.isAlive() && handoff.offer(new Job(CLOSE, -1), HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { // Drain the exit acknowledgement so close() returns only after the thread is done. completion.poll(HANDOFF_TIMEOUT_SECONDS, TimeUnit.SECONDS); } @@ -123,18 +166,18 @@ private void loop() { tid = ReferenceChainTestHooks.getTid(); try { while (true) { - Runnable task = handoff.take(); - if (task == CLOSE) { - completion.offer(new Object()); // exit acknowledgement; no further work arrives + Job job = handoff.take(); + if (job.task == CLOSE) { + completion.offer(new Outcome(job.generation, new Object())); // exit acknowledgement; no further work arrives return; } Object outcome = null; try { - task.run(); + job.task.run(); } catch (RuntimeException | Error t) { outcome = t; } - completion.offer(outcome != null ? outcome : new Object()); + completion.offer(new Outcome(job.generation, outcome != null ? outcome : new Object())); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); From 8a48963f70f5024c476ee49197bbdf1078add0cf Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 18:26:08 +0200 Subject: [PATCH 25/28] Fail loud on missing HeapLiveObject accessors; single evidence rank Both leak-correlation scenarios silently degraded a missing leakTag (or, in LeakTagCorrelation, leakTagIncarnation) accessor to -1, yielding an empty correlation with no diagnostic - the failure mode the fail-loud guards were added to prevent. Both scenarios now throw when either identity accessor is absent. Consolidate LeakTagCorrelationScenario's owner-verified/in-pool/out-of-pool ranking into evidenceRank() (used by evidenceScore(), the within-scan candidate selection, and cross-referenced by the marker cascade) so the three sites cannot drift apart. Document that ThreadLocalLeakScenario's keepBest has nothing to merge from a losing scan (no transient/noise evidence fields in that scenario). --- .../LeakTagCorrelationScenario.java | 39 +++++++++++++++---- .../ThreadLocalLeakScenario.java | 21 +++++++--- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index 961e9a272c..fecf64a0fa 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -407,6 +407,10 @@ public static void run(JavaProfiler profiler, String startCommand, Path scratchD System.out.println(NOISE_CHAIN_MARKER + best.noiseChains); return; } + // Marker order mirrors the evidence ranking in evidenceScore() below: + // out-of-pool (0) < unverified in-pool (1) < owner-verified in-pool (2) - + // report the weakest evidence first, and keep this cascade in lockstep with + // evidenceScore() if the ranking ever changes. if (best == null || best.payloadChain == null) { System.out.println(NOT_FOUND_MARKER); return; @@ -550,7 +554,15 @@ private static int evidenceScore(ScanResult scan) { if (scan.payloadChain == null) { return -1; } - return scan.payloadChainInPool ? (scan.payloadChainOwnerVerified ? 2 : 1) : 0; + return evidenceRank(scan.payloadChainInPool, scan.payloadChainOwnerVerified); + } + + // The single definition of the scan-evidence ranking: owner-verified in-pool (2) + // > unverified in-pool (1) > out-of-pool (0). scan()'s within-dump candidate + // selection and keepBest()'s cross-dump merge both derive from this, so a + // ranking change cannot leave one site disagreeing with the others. + private static int evidenceRank(boolean inPool, boolean ownerVerified) { + return inPool ? (ownerVerified ? 2 : 1) : 0; } private static void mergeDistinct(List into, List from) { @@ -625,9 +637,9 @@ private static ScanResult scan(Path scratchDumpPath, Map liveOwne // in-pool chain, then any in-pool one, over an out-of-pool one seen earlier. boolean ownerVerified = inPool && (liveOwners == null || liveOwners.containsKey(targetTag)); - int score = inPool ? (ownerVerified ? 2 : 1) : 0; + int score = evidenceRank(inPool, ownerVerified); int currentScore = result.payloadChain == null ? -1 - : result.payloadChainInPool ? (result.payloadChainOwnerVerified ? 2 : 1) : 0; + : evidenceRank(result.payloadChainInPool, result.payloadChainOwnerVerified); if (score > currentScore) { List chain = new ArrayList<>(rawChain.length); for (Object element : rawChain) { @@ -657,11 +669,24 @@ private static ScanResult scan(Path scratchDumpPath, Map liveOwne ReferenceChainAssertions.findAccessor(type, "objectClass"); IMemberAccessor incarnationAccessor = ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); + // Fail loud if either identity field is missing, mirroring + // ThreadLocalLeakScenario's guard: without the tag every live object + // reads -1 and never matches, without the incarnation the match + // degrades to tag-only - both must surface as errors, not as an empty + // correlation with no diagnostic. + if (leakTagAccessor == null) { + throw new IllegalStateException( + "datadog.HeapLiveObject events have no leakTag accessor"); + } + if (incarnationAccessor == null) { + throw new IllegalStateException( + "datadog.HeapLiveObject events have no leakTagIncarnation accessor"); + } for (IItem item : iterable) { - long leakTag = leakTagAccessor != null - ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; - long incarnation = incarnationAccessor != null - ? ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)) : -1; + long leakTag = + ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)); + long incarnation = + ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)); // Leak tags are recycled: the tag alone is ambiguous across chunks. if (leakTag != result.payloadChain.targetTag || incarnation != result.payloadChainIncarnation) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java index 1fe37317f8..4d1fe76424 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -332,6 +332,11 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { if (scan == null) { return best; } + // Unlike LeakTagCorrelationScenario.keepBest, the losing scan has nothing + // to merge: this scenario's ScanResult carries no transient/noise evidence + // (that is LeakTagCorrelation's contract), and a scan without a payload + // chain has no other populated field worth keeping. If evidence fields are + // ever added here, merge them like LeakTagCorrelationScenario's mergeDistinct. if (best == null || best.payloadChain == null) { return scan; } @@ -436,16 +441,22 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { ReferenceChainAssertions.findAccessor(type, "objectClass"); IMemberAccessor incarnationAccessor = ReferenceChainAssertions.findAccessor(type, "leakTagIncarnation"); - // Fail loud if the field is missing: without it every live object - // reads -1 and the correlation below would degrade to tag-only - // matching, which a recycled tag's stale chunk could satisfy. + // Fail loud if either identity field is missing: without the tag, every + // live object reads -1 and never matches the chain's targetTag; without + // the incarnation, the match degrades to tag-only, which a recycled + // tag's stale chunk could satisfy. Both must surface as errors, not as + // an empty correlation with no diagnostic. if (incarnationAccessor == null) { throw new IllegalStateException( "datadog.HeapLiveObject events have no leakTagIncarnation accessor"); } + if (leakTagAccessor == null) { + throw new IllegalStateException( + "datadog.HeapLiveObject events have no leakTag accessor"); + } for (IItem item : iterable) { - long leakTag = leakTagAccessor != null - ? ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)) : -1; + long leakTag = + ReferenceChainAssertions.numberValue(leakTagAccessor.getMember(item)); long incarnation = ReferenceChainAssertions.numberValue(incarnationAccessor.getMember(item)); // Leak tags are recycled: the tag alone is ambiguous across chunks. From 9e3002b2144a9a4be811f7b82f00d649b4eaa7ed Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 19:46:57 +0200 Subject: [PATCH 26/28] Value-initialize _leak_tag_in_use alongside the ownership stamps start() rebuilds the leak-tag pool by comparing every _leak_tag_in_use[i] against the table's owners; the singleton is static-storage and zero-initialized regardless, but a non-statically constructed tracker would read indeterminate bools there. Value-init the array in the constructor's init list, matching _leak_tag_incarnation. --- ddprof-lib/src/main/cpp/livenessTracker.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h index 319739ec86..20afe41418 100644 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ b/ddprof-lib/src/main/cpp/livenessTracker.h @@ -857,6 +857,11 @@ class alignas(alignof(SpinLock)) LivenessTracker { _klass_population_size(0), _klass_count_scratch_size(0), _last_class_map_generation(0), _leak_tag_free_count(LEAK_TAG_POOL_SIZE), + // Value-initialized alongside the stamps: start() rebuilds the pool by + // comparing every _leak_tag_in_use[i] against the table's owners, so a + // non-statically constructed tracker must read all-false there. (The + // singleton is static-storage and zero-initialized regardless.) + _leak_tag_in_use(), // value-initialized: every slot starts free _leak_tag_incarnation() {} // value-initialized: every stamp starts at 0 // Ownership-period stamp of leak tag `tag` (see _leak_tag_incarnation), or 0 From e68476f2f372e4c4349b5827ca44c14edf698cee Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 20:05:24 +0200 Subject: [PATCH 27/28] Harden PersistentAllocator publication and sidecar diagnostics The worker's completion.offer() could silently drop THIS task's own outcome: with one slot, a stale entry left by an abandoned (timed-out) run() can still occupy the queue when the current task finishes, and the dropped outcome turns a completed task into the caller's 30s poll timeout. Use put() instead - the caller's generation loop drains the stale entry, unblocking the worker; the same applies to close()'s exit acknowledgement, which close()'s drain unblocks. keepBest(): drop the dead other != null check (both scans are non-null after the early returns). The roundtrip sidecar readers now name the offending key and value instead of surfacing a bare NumberFormatException (or "null" for a missing key). --- .../LeakTagCorrelationScenario.java | 15 +++---- .../referencechains/PersistentAllocator.java | 14 ++++++- .../ReferenceChainJfrParserTest.java | 39 +++++++++++++------ 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java index fecf64a0fa..346728f387 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakTagCorrelationScenario.java @@ -13,6 +13,7 @@ import org.openjdk.jmc.common.item.IItemIterable; import org.openjdk.jmc.common.item.IMemberAccessor; import org.openjdk.jmc.common.item.IType; +import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; import java.io.InputStream; @@ -488,7 +489,7 @@ private static boolean hasByteArrayChain(Path scratchDumpPath, long tag, long in events = JfrLoaderToolkit.loadEvents(in); } IItemCollection chains = events.apply( - org.openjdk.jmc.common.item.ItemFilters.type("datadog.ReferenceChain")); + ItemFilters.type("datadog.ReferenceChain")); for (IItemIterable iterable : chains) { IType type = iterable.getType(); IMemberAccessor chainAccessor = ReferenceChainAssertions.findAccessor(type, "chain"); @@ -543,10 +544,10 @@ private static ScanResult keepBest(ScanResult best, ScanResult scan) { ScanResult result = scanScore > bestScore || (scanScore == bestScore && scan.payloadChain != null) ? scan : best; ScanResult other = result == scan ? best : scan; - if (other != null) { - mergeDistinct(result.transientViolations, other.transientViolations); - mergeDistinct(result.noiseChains, other.noiseChains); - } + // Both non-null here: the early returns above already handled a null scan + // (kept best) and a null best (kept scan), so the loser always exists. + mergeDistinct(result.transientViolations, other.transientViolations); + mergeDistinct(result.noiseChains, other.noiseChains); return result; } @@ -595,7 +596,7 @@ private static ScanResult scan(Path scratchDumpPath, Map liveOwne ScanResult result = new ScanResult(); IItemCollection chains = events.apply( - org.openjdk.jmc.common.item.ItemFilters.type("datadog.ReferenceChain")); + ItemFilters.type("datadog.ReferenceChain")); for (IItemIterable iterable : chains) { IType type = iterable.getType(); IMemberAccessor chainAccessor = ReferenceChainAssertions.findAccessor(type, "chain"); @@ -660,7 +661,7 @@ private static ScanResult scan(Path scratchDumpPath, Map liveOwne if (result.payloadChainInPool) { IItemCollection liveObjects = events.apply( - org.openjdk.jmc.common.item.ItemFilters.type("datadog.HeapLiveObject")); + ItemFilters.type("datadog.HeapLiveObject")); for (IItemIterable iterable : liveObjects) { IType type = iterable.getType(); IMemberAccessor leakTagAccessor = diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java index 94441b1b43..02f7b08b9d 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/PersistentAllocator.java @@ -168,7 +168,11 @@ private void loop() { while (true) { Job job = handoff.take(); if (job.task == CLOSE) { - completion.offer(new Outcome(job.generation, new Object())); // exit acknowledgement; no further work arrives + // put (blocking), not offer: after an abandoned run() the slot can still + // hold that run's stale outcome, and a dropped exit acknowledgement would + // leave close()'s poll draining the stale entry instead. close() always + // drains exactly one entry after the handoff, which unblocks this put. + completion.put(new Outcome(job.generation, new Object())); // exit acknowledgement; no further work arrives return; } Object outcome = null; @@ -177,7 +181,13 @@ private void loop() { } catch (RuntimeException | Error t) { outcome = t; } - completion.offer(new Outcome(job.generation, outcome != null ? outcome : new Object())); + // put (blocking), not offer: with one slot, a stale outcome left by an + // abandoned (timed-out) run() can still occupy the queue when this + // task finishes, and a dropped outcome would turn a completed task into + // the caller's 30s poll timeout. The caller's generation loop drains the + // stale entry (discard-by-generation), which unblocks this put; if the + // caller already gave up, the next run()'s clear() or close()'s poll does. + completion.put(new Outcome(job.generation, outcome != null ? outcome : new Object())); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java index 555ff34cc7..30e7537c53 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainJfrParserTest.java @@ -15,6 +15,7 @@ import org.openjdk.jmc.common.item.IMemberAccessor; import org.openjdk.jmc.common.item.IType; import org.openjdk.jmc.common.item.ItemFilters; +import org.openjdk.jmc.common.unit.IQuantity; import org.openjdk.jmc.flightrecorder.CouldNotLoadRecordingException; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; @@ -26,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -76,7 +78,7 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() "Expected " + expectedPath + " to exist - run " + ":ddprof-lib:gtestDebug_referenceChainJfrRoundtrip_ut first " + "(referenceChainJfrRoundtrip_ut.cpp produces this sidecar)."); - java.util.Properties expected = new java.util.Properties(); + Properties expected = new Properties(); try (InputStream in = Files.newInputStream(expectedPath)) { expected.load(in); } @@ -147,8 +149,8 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() Object v = targetTagAccessor.getMember(item); if (v instanceof Number) { targetTag = ((Number) v).longValue(); - } else if (v instanceof org.openjdk.jmc.common.unit.IQuantity) { - targetTag = ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); + } else if (v instanceof IQuantity) { + targetTag = ((IQuantity) v).longValue(); } } leakTagIncarnation = longValue(incarnationAccessor.getMember(item)); @@ -156,8 +158,8 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() Object v = depthAccessor.getMember(item); if (v instanceof Number) { depth = ((Number) v).intValue(); - } else if (v instanceof org.openjdk.jmc.common.unit.IQuantity) { - depth = (int) ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); + } else if (v instanceof IQuantity) { + depth = (int) ((IQuantity) v).longValue(); } } break; // referenceChainJfrRoundtrip_ut.cpp writes exactly one event @@ -187,19 +189,34 @@ public void chainFieldResolvesToSeededClassNamesInLeafToRootOrder() "leakTagIncarnation should carry the event's ownership stamp the gtest seeded"); } - private static int expectedInt(java.util.Properties expected, String key) { - return Integer.parseInt(expected.getProperty(key)); + // Both sidecar readers report the offending key with the value: a bare parse + // exception names neither, and a MISSING key surfaces as a null parse whose + // built-in message ("null") says nothing about which sidecar entry is absent. + private static int expectedInt(Properties expected, String key) { + String value = expected.getProperty(key); + try { + return Integer.parseInt(value); + } catch (RuntimeException e) { + throw new IllegalStateException( + "malformed or missing '" + key + "' in the roundtrip sidecar (value: " + value + ")", e); + } } - private static long expectedLong(java.util.Properties expected, String key) { - return Long.parseLong(expected.getProperty(key)); + private static long expectedLong(Properties expected, String key) { + String value = expected.getProperty(key); + try { + return Long.parseLong(value); + } catch (RuntimeException e) { + throw new IllegalStateException( + "malformed or missing '" + key + "' in the roundtrip sidecar (value: " + value + ")", e); + } } private static long longValue(Object v) { if (v instanceof Number) { return ((Number) v).longValue(); - } else if (v instanceof org.openjdk.jmc.common.unit.IQuantity) { - return ((org.openjdk.jmc.common.unit.IQuantity) v).longValue(); + } else if (v instanceof IQuantity) { + return ((IQuantity) v).longValue(); } return -1; } From 4feba62ddb0d139b58b39c55ce29cc15700c7830 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Mon, 19 Oct 2026 20:05:24 +0200 Subject: [PATCH 28/28] Replace inline fully-qualified names with imports Code bodies referenced java.util.Properties, org.openjdk.jmc.common.{item,unit} types, java.lang.reflect.Method and java.lang.management.ManagementFactory inline; import them and use the simple names instead. Javadoc prose and message strings naming JDK internals (chain path descriptions) are untouched. --- .../com/datadoghq/profiler/AbstractProfilerTest.java | 3 ++- .../referencechains/LeakingCacheScenario.java | 12 ++++++++---- .../referencechains/ReferenceChainTrackingTest.java | 3 ++- .../referencechains/ThreadLocalLeakScenario.java | 5 +++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java index a157ae1195..acecb8fef4 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java @@ -5,6 +5,7 @@ package com.datadoghq.profiler; +import java.lang.management.ManagementFactory; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -147,7 +148,7 @@ protected final boolean isDebugBuild() { private static long getPid() { try { - String name = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); + String name = ManagementFactory.getRuntimeMXBean().getName(); return Long.parseLong(name.split("@")[0]); } catch (NumberFormatException e) { return 0L; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java index 7d91bdc90e..f23d5f2312 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/LeakingCacheScenario.java @@ -8,7 +8,11 @@ import com.datadoghq.profiler.JavaProfiler; import com.datadoghq.profiler.ReferenceChainTestHooks; import org.openjdk.jmc.common.IMCType; +import org.openjdk.jmc.common.item.IItem; import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.common.item.IType; import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; @@ -243,15 +247,15 @@ private static void debugDumpAllChainLeaves(Path scratchDumpPath) { IItemCollection abandoned = events.apply(ItemFilters.type("datadog.ReferenceChainAbandoned")); long totalAbandoned = abandoned.stream().mapToLong(it -> it.getItemCount()).sum(); System.out.println("[debug] datadog.ReferenceChainAbandoned total events: " + totalAbandoned); - for (org.openjdk.jmc.common.item.IItemIterable iterable : chains) { - org.openjdk.jmc.common.item.IType type = iterable.getType(); - org.openjdk.jmc.common.item.IMemberAccessor chainAccessor = + for (IItemIterable iterable : chains) { + IType type = iterable.getType(); + IMemberAccessor chainAccessor = ReferenceChainAssertions.findAccessor(type, "chain"); if (chainAccessor == null) { System.out.println("[debug] no chain accessor found"); continue; } - for (org.openjdk.jmc.common.item.IItem item : iterable) { + for (IItem item : iterable) { Object chainValue = chainAccessor.getMember(item); if (chainValue instanceof Object[]) { Object[] raw = (Object[]) chainValue; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java index 84d01c6e9b..867e02af18 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ReferenceChainTrackingTest.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.lang.reflect.Method; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -84,7 +85,7 @@ public class ReferenceChainTrackingTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { String testName = testInfo != null - ? testInfo.getTestMethod().map(java.lang.reflect.Method::getName).orElse("") + ? testInfo.getTestMethod().map(Method::getName).orElse("") : ""; if ("shouldReconstructReferrerChainToGcRoot".equals(testName) || "shouldReconstructReferrerChainThroughUnboundedCacheLeak".equals(testName)) { diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java index 4d1fe76424..5f7fb799bf 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/referencechains/ThreadLocalLeakScenario.java @@ -13,6 +13,7 @@ import org.openjdk.jmc.common.item.IItemIterable; import org.openjdk.jmc.common.item.IMemberAccessor; import org.openjdk.jmc.common.item.IType; +import org.openjdk.jmc.common.item.ItemFilters; import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; import java.io.InputStream; @@ -370,7 +371,7 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { ScanResult result = new ScanResult(); IItemCollection chains = events.apply( - org.openjdk.jmc.common.item.ItemFilters.type("datadog.ReferenceChain")); + ItemFilters.type("datadog.ReferenceChain")); for (IItemIterable iterable : chains) { IType type = iterable.getType(); IMemberAccessor chainAccessor = @@ -432,7 +433,7 @@ private static ScanResult scan(Path scratchDumpPath) throws Exception { if (result.payloadChainInPool) { IItemCollection liveObjects = events.apply( - org.openjdk.jmc.common.item.ItemFilters.type("datadog.HeapLiveObject")); + ItemFilters.type("datadog.HeapLiveObject")); for (IItemIterable iterable : liveObjects) { IType type = iterable.getType(); IMemberAccessor leakTagAccessor =