From 996e627493f777d32c5bfd01913bc0fa2b7bbf28 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 20 Oct 2026 11:52:38 +0200 Subject: [PATCH 1/5] Add USDT output for native allocations --- ddprof-lib/src/main/cpp/arguments.cpp | 14 + ddprof-lib/src/main/cpp/arguments.h | 10 + ddprof-lib/src/main/cpp/mallocTracer.cpp | 139 ++++- ddprof-lib/src/main/cpp/mallocTracer.h | 29 +- .../src/main/cpp/nativeAllocationUsdt.cpp | 81 +++ .../src/main/cpp/nativeAllocationUsdt.h | 26 + ddprof-lib/src/main/cpp/profiler.cpp | 2 +- .../src/main/cpp/vendor/libbpf/README.md | 16 + ddprof-lib/src/main/cpp/vendor/libbpf/usdt.h | 545 ++++++++++++++++++ ddprof-lib/src/test/cpp/arguments_ut.cpp | 70 +++ .../src/test/cpp/mallocTracer_usdt_ut.cpp | 251 ++++++++ doc/architecture/NativeMemoryProfiling.md | 28 +- 12 files changed, 1190 insertions(+), 21 deletions(-) create mode 100644 ddprof-lib/src/main/cpp/nativeAllocationUsdt.cpp create mode 100644 ddprof-lib/src/main/cpp/nativeAllocationUsdt.h create mode 100644 ddprof-lib/src/main/cpp/vendor/libbpf/README.md create mode 100644 ddprof-lib/src/main/cpp/vendor/libbpf/usdt.h create mode 100644 ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp diff --git a/ddprof-lib/src/main/cpp/arguments.cpp b/ddprof-lib/src/main/cpp/arguments.cpp index 82db261aa5..f453ff95e4 100644 --- a/ddprof-lib/src/main/cpp/arguments.cpp +++ b/ddprof-lib/src/main/cpp/arguments.cpp @@ -425,6 +425,20 @@ Error Arguments::parse(const char *args) { msg = "nativemem must be >= 0"; } + CASE("nativememoutput") + _nativemem_output_set = true; + if (value == NULL || *value == '\0') { + msg = "nativememoutput requires jfr, usdt, or both"; + } else if (strcasecmp(value, "jfr") == 0) { + _nativemem_output = NATIVE_MEMORY_JFR; + } else if (strcasecmp(value, "usdt") == 0) { + _nativemem_output = NATIVE_MEMORY_USDT; + } else if (strcasecmp(value, "both") == 0) { + _nativemem_output = NATIVE_MEMORY_BOTH; + } else { + msg = "Invalid nativememoutput value"; + } + CASE("natsock") if (value != NULL) { _nativesocket_interval = parseUnits(value, NANOS); diff --git a/ddprof-lib/src/main/cpp/arguments.h b/ddprof-lib/src/main/cpp/arguments.h index d6222f9157..04037fe246 100644 --- a/ddprof-lib/src/main/cpp/arguments.h +++ b/ddprof-lib/src/main/cpp/arguments.h @@ -112,6 +112,12 @@ enum Ring { RING_ANY = RING_KERNEL | RING_USER, }; +enum NativeMemoryOutput { + NATIVE_MEMORY_JFR, + NATIVE_MEMORY_USDT, + NATIVE_MEMORY_BOTH +}; + enum Style { STYLE_SIMPLE = 1, STYLE_DOTTED = 2, @@ -254,6 +260,8 @@ class Arguments { u8 _reference_chains_tuned_mask; // Explicit opt-in for the legacy whole-graph JVMTI FollowReferences walk. long _nativemem; + NativeMemoryOutput _nativemem_output; + bool _nativemem_output_set; int _jstackdepth; int _safe_mode; StackWalkFeatures _features; @@ -305,6 +313,8 @@ class Arguments { _reference_chains_first_pass_budget(DEFAULT_REFERENCE_CHAINS_FIRST_PASS_BUDGET), _reference_chains_tuned_mask(0), _nativemem(-1), + _nativemem_output(NATIVE_MEMORY_JFR), + _nativemem_output_set(false), _jstackdepth(DEFAULT_JSTACKDEPTH), _safe_mode(0), _features{1, 1, 1, 1, 1, 1}, diff --git a/ddprof-lib/src/main/cpp/mallocTracer.cpp b/ddprof-lib/src/main/cpp/mallocTracer.cpp index 0ae8e6b914..0e89f16d46 100644 --- a/ddprof-lib/src/main/cpp/mallocTracer.cpp +++ b/ddprof-lib/src/main/cpp/mallocTracer.cpp @@ -14,6 +14,7 @@ #include "guards.h" #include "libraries.h" #include "mallocTracer.h" +#include "nativeAllocationUsdt.h" #include "os.h" #include "pidController.h" #include "profiler.h" @@ -87,6 +88,7 @@ u64 MallocTracer::_configured_interval; volatile u64 MallocTracer::_sample_count; volatile u64 MallocTracer::_last_config_update_ts; volatile bool MallocTracer::_running = false; +NativeMemoryOutput MallocTracer::_output = NATIVE_MEMORY_JFR; PidController MallocTracer::_pid(MallocTracer::TARGET_SAMPLES_PER_WINDOW, 31, 511, 3, MallocTracer::CONFIG_UPDATE_CHECK_PERIOD_SECS, 15); @@ -102,6 +104,20 @@ static pthread_t _current_thread; static volatile bool _nested_malloc = false; static volatile bool _nested_posix_memalign = false; +#ifdef UNIT_TEST +static MallocTracer::AcquireCurrentTestHook _test_acquire_current; +static MallocTracer::CriticalSectionTestHook _test_critical_section; +static MallocTracer::ShouldSampleTestHook _test_should_sample; +static MallocTracer::JfrSinkTestHook _test_jfr_sink; +static MallocTracer::UsdtActiveTestHook _test_usdt_active; +static MallocTracer::UsdtEmitTestHook _test_usdt_emit; +static MallocTracer::StageTestHook _test_stage; + +static inline void testStage(const char* stage) { + if (_test_stage != NULL) _test_stage(stage); +} +#endif + // Test if calloc() implementation calls malloc() static void* nested_malloc_hook(size_t size) { if (pthread_self() == _current_thread) { @@ -313,15 +329,33 @@ void MallocTracer::maybeRecord(void* ret, size_t size) { if (running() && ret && size) { // Even we are not in a signal handler, we cannot malloc or // we may get into indefinite loop +#ifdef UNIT_TEST + ProfiledThread* prof_thread = _test_acquire_current == NULL + ? ProfiledThread::acquireCurrent() : _test_acquire_current(); + testStage("acquire"); +#else ProfiledThread* prof_thread = ProfiledThread::acquireCurrent(); +#endif if (prof_thread == nullptr) { Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL); return; } CriticalSection cs(prof_thread); +#ifdef UNIT_TEST + bool entered = _test_critical_section == NULL + ? cs.entered() : _test_critical_section(cs.entered()); + testStage("critical-section"); + if (entered) { + bool sampled = _test_should_sample == NULL + ? shouldSample(size) : _test_should_sample(size); + testStage("sampling"); + if (sampled) recordMalloc(ret, size, prof_thread->tid()); + } +#else if (cs.entered() && shouldSample(size)) { recordMalloc(ret, size, prof_thread->tid()); } +#endif } } @@ -347,18 +381,53 @@ void MallocTracer::recordMalloc(void* address, size_t size, int tid) { SAMPLER_PERF_PROBE(SP_NATIVEMEM); u64 current_interval = __atomic_load_n(&_interval, __ATOMIC_ACQUIRE); - MallocEvent event; - event._start_time = TSC::ticks(); - event._address = (uintptr_t)address; - event._size = size; - // _interval == 0 means sample every allocation; weight is 1.0. - if (size == 0 || current_interval <= 1) { - event._weight = 1.0f; - } else { - event._weight = (float)(1.0 / (1.0 - exp(-(double)size / (double)current_interval))); + if (_output != NATIVE_MEMORY_USDT) { + MallocEvent event; + event._start_time = TSC::ticks(); + event._address = (uintptr_t)address; + event._size = size; + // _interval == 0 means sample every allocation; weight is 1.0. + if (size == 0 || current_interval <= 1) { + event._weight = 1.0f; + } else { + event._weight = (float)(1.0 / (1.0 - exp(-(double)size / (double)current_interval))); + } + +#ifdef UNIT_TEST + testStage("jfr"); + if (_test_jfr_sink != NULL) { + _test_jfr_sink(address, size, tid, &event); + } else { + Profiler::instance()->recordSample(NULL, size, tid, BCI_NATIVE_MALLOC, 0, &event); + } +#else + Profiler::instance()->recordSample(NULL, size, tid, BCI_NATIVE_MALLOC, 0, &event); +#endif } - Profiler::instance()->recordSample(NULL, size, tid, BCI_NATIVE_MALLOC, 0, &event); + bool usdt_active = false; + if (_output != NATIVE_MEMORY_JFR && size <= (size_t)INT64_MAX) { +#ifdef UNIT_TEST + usdt_active = _test_usdt_active == NULL + ? isNativeAllocationUsdtEnabled() : _test_usdt_active(); +#else + usdt_active = isNativeAllocationUsdtEnabled(); +#endif + } + if (usdt_active) { + uint64_t weighted_bytes = nativeAllocationUsdtWeight(size, current_interval); +#ifdef UNIT_TEST + testStage("usdt"); + if (_test_usdt_emit != NULL) { + _test_usdt_emit(address, (uint64_t)size, weighted_bytes); + } else { + emitNativeAllocationUsdt(address, (uint64_t)size, weighted_bytes); + } +#else + emitNativeAllocationUsdt(address, (uint64_t)size, + weighted_bytes); +#endif + } u64 current_samples = __atomic_add_fetch(&_sample_count, 1, __ATOMIC_RELAXED); if ((current_samples % TARGET_SAMPLES_PER_WINDOW) == 0) { @@ -375,9 +444,13 @@ void MallocTracer::recordMalloc(void* address, size_t size, int tid) { } } } +#ifdef UNIT_TEST + testStage("accounting"); +#endif } Error MallocTracer::start(Arguments& args) { + _output = args._nativemem_output; _configured_interval = args._nativemem > 0 ? args._nativemem : 0; __atomic_store_n(&_interval, _configured_interval, __ATOMIC_RELEASE); __atomic_store_n(&_bytes_until_sample, @@ -403,8 +476,54 @@ Error MallocTracer::start(Arguments& args) { return Error::OK; } +Error MallocTracer::check(Arguments& args) { + if (args._nativemem_output_set && args._nativemem < 0) { + return Error("nativememoutput requires nativemem"); + } +#if !defined(__linux__) + if (args._nativemem_output != NATIVE_MEMORY_JFR) { + return Error("nativememoutput=usdt and both are supported only on Linux"); + } +#endif + return Error::OK; +} + void MallocTracer::stop() { // Ideally, we should reset original malloc entries, but it's not currently safe // in the view of library unloading. Consider using dl_iterate_phdr. __atomic_store_n(&_running, false, __ATOMIC_RELEASE); } + +#ifdef UNIT_TEST +void MallocTracer::setTestHooks(AcquireCurrentTestHook acquire, + CriticalSectionTestHook critical_section, + ShouldSampleTestHook should_sample, + JfrSinkTestHook jfr_sink, + UsdtActiveTestHook usdt_active, + UsdtEmitTestHook usdt_emit, + StageTestHook stage) { + _test_acquire_current = acquire; + _test_critical_section = critical_section; + _test_should_sample = should_sample; + _test_jfr_sink = jfr_sink; + _test_usdt_active = usdt_active; + _test_usdt_emit = usdt_emit; + _test_stage = stage; +} + +void MallocTracer::resetTestHooks() { + setTestHooks(NULL, NULL, NULL, NULL, NULL, NULL, NULL); + configureForTest(NATIVE_MEMORY_JFR, 0, false); +} + +void MallocTracer::configureForTest(NativeMemoryOutput output, u64 interval, + bool running) { + _output = output; + _configured_interval = interval; + __atomic_store_n(&_interval, interval, __ATOMIC_RELEASE); + __atomic_store_n(&_bytes_until_sample, 0, __ATOMIC_RELEASE); + __atomic_store_n(&_sample_count, (u64)0, __ATOMIC_RELEASE); + __atomic_store_n(&_last_config_update_ts, OS::nanotime(), __ATOMIC_RELEASE); + __atomic_store_n(&_running, running, __ATOMIC_RELEASE); +} +#endif diff --git a/ddprof-lib/src/main/cpp/mallocTracer.h b/ddprof-lib/src/main/cpp/mallocTracer.h index f89bda4d88..08a2be614d 100644 --- a/ddprof-lib/src/main/cpp/mallocTracer.h +++ b/ddprof-lib/src/main/cpp/mallocTracer.h @@ -13,6 +13,8 @@ #include "mutex.h" #include "pidController.h" +class ProfiledThread; + // Manages GOT-patching for malloc interception across all loaded native libraries. class MallocHooker { private: @@ -41,6 +43,7 @@ class MallocTracer : public Engine { static const int TARGET_SAMPLES_PER_WINDOW = 100; static volatile bool _running; + static NativeMemoryOutput _output; static PidController _pid; static u64 nextPoissonInterval(); @@ -53,6 +56,7 @@ class MallocTracer : public Engine { } Error start(Arguments& args); + Error check(Arguments& args) override; void stop(); static inline bool running() { @@ -65,9 +69,30 @@ class MallocTracer : public Engine { static void recordMalloc(void* address, size_t size, int tid); - // Entry point for the malloc hooks (mallocTracer.cpp): gates on the - // sampling decision before touching thread-local state. + // Entry point for malloc hooks. Thread-local and critical-section guards + // must be acquired before the shared sampling state is touched. static void maybeRecord(void* ret, size_t size); + +#ifdef UNIT_TEST + using AcquireCurrentTestHook = ProfiledThread* (*)(); + using CriticalSectionTestHook = bool (*)(bool); + using ShouldSampleTestHook = bool (*)(size_t); + using JfrSinkTestHook = void (*)(void*, size_t, int, MallocEvent*); + using UsdtActiveTestHook = bool (*)(); + using UsdtEmitTestHook = void (*)(void*, uint64_t, uint64_t); + using StageTestHook = void (*)(const char*); + + static void setTestHooks(AcquireCurrentTestHook acquire, + CriticalSectionTestHook critical_section, + ShouldSampleTestHook should_sample, + JfrSinkTestHook jfr_sink, + UsdtActiveTestHook usdt_active, + UsdtEmitTestHook usdt_emit, + StageTestHook stage); + static void resetTestHooks(); + static void configureForTest(NativeMemoryOutput output, u64 interval, + bool running); +#endif }; #endif // _MALLOCTRACER_H diff --git a/ddprof-lib/src/main/cpp/nativeAllocationUsdt.cpp b/ddprof-lib/src/main/cpp/nativeAllocationUsdt.cpp new file mode 100644 index 0000000000..5fbf18bead --- /dev/null +++ b/ddprof-lib/src/main/cpp/nativeAllocationUsdt.cpp @@ -0,0 +1,81 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "nativeAllocationUsdt.h" + +#include +#include + +#if defined(__linux__) +#define USDT_ARG_CONSTRAINT r +#include "vendor/libbpf/usdt.h" + +alignas(2) USDT_DEFINE_SEMA(ddheap_alloc); + +bool isNativeAllocationUsdtEnabled() noexcept { + return __atomic_load_n(&USDT_SEMA(ddheap_alloc).active, + __ATOMIC_RELAXED) != 0; +} + +#if defined(__GNUC__) && !defined(__clang__) +__attribute__((noinline, noclone, used)) +#else +__attribute__((noinline, used)) +#endif +void emitNativeAllocationUsdt(void* address, uint64_t size, + uint64_t weighted_bytes) noexcept { +#if defined(__x86_64__) + register void* probe_address asm("rdi") = address; + register uint64_t probe_size asm("rsi") = size; + register uint64_t probe_weight asm("rdx") = weighted_bytes; +#elif defined(__aarch64__) + register void* probe_address asm("x0") = address; + register uint64_t probe_size asm("x1") = size; + register uint64_t probe_weight asm("x2") = weighted_bytes; +#else +#error "Native allocation USDT supports only Linux x86-64 and AArch64" +#endif + USDT_WITH_EXPLICIT_SEMA(ddheap_alloc, ddheap, alloc, + probe_address, probe_size, probe_weight); +} + +#ifdef UNIT_TEST +void setNativeAllocationUsdtSemaphoreForTest(uint16_t value) noexcept { + __atomic_store_n(&USDT_SEMA(ddheap_alloc).active, value, + __ATOMIC_RELAXED); +} +#endif + +#else + +bool isNativeAllocationUsdtEnabled() noexcept { + return false; +} + +void emitNativeAllocationUsdt(void*, uint64_t, uint64_t) noexcept { +} + +#ifdef UNIT_TEST +void setNativeAllocationUsdtSemaphoreForTest(uint16_t) noexcept { +} +#endif + +#endif + +uint64_t nativeAllocationUsdtWeight(size_t size, uint64_t interval) noexcept { + if (size == 0 || interval <= 1) { + return size; + } + + double probability = -expm1(-(double)size / (double)interval); + double weighted_bytes = (double)size / probability; + if (!std::isfinite(weighted_bytes) || weighted_bytes >= (double)INT64_MAX) { + return (uint64_t)INT64_MAX; + } + if (weighted_bytes <= 1.0) { + return 1; + } + return (uint64_t)llround(weighted_bytes); +} diff --git a/ddprof-lib/src/main/cpp/nativeAllocationUsdt.h b/ddprof-lib/src/main/cpp/nativeAllocationUsdt.h new file mode 100644 index 0000000000..6dbecd02e5 --- /dev/null +++ b/ddprof-lib/src/main/cpp/nativeAllocationUsdt.h @@ -0,0 +1,26 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _NATIVEALLOCATIONUSDT_H +#define _NATIVEALLOCATIONUSDT_H + +#include +#include + +// Returns a point-in-time indication that at least one consumer is attached. +bool isNativeAllocationUsdtEnabled() noexcept; + +// Emits ddheap:alloc. Call only after isNativeAllocationUsdtEnabled(). +void emitNativeAllocationUsdt(void* address, uint64_t size, + uint64_t weighted_bytes) noexcept; + +// Converts this sampler's object-count weight into ddheap's weighted-byte ABI. +uint64_t nativeAllocationUsdtWeight(size_t size, uint64_t interval) noexcept; + +#ifdef UNIT_TEST +void setNativeAllocationUsdtSemaphoreForTest(uint16_t value) noexcept; +#endif + +#endif // _NATIVEALLOCATIONUSDT_H diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 778cc5bbdd..1c5ab4e168 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1974,7 +1974,7 @@ Error Profiler::check(Arguments &args) { _alloc_engine = selectAllocEngine(args); error = _alloc_engine->check(args); } - if (!error && args._nativemem >= 0) { + if (!error && (args._nativemem >= 0 || args._nativemem_output_set)) { error = malloc_tracer.check(args); } if (!error && args._nativesocket) { diff --git a/ddprof-lib/src/main/cpp/vendor/libbpf/README.md b/ddprof-lib/src/main/cpp/vendor/libbpf/README.md new file mode 100644 index 0000000000..a231024104 --- /dev/null +++ b/ddprof-lib/src/main/cpp/vendor/libbpf/README.md @@ -0,0 +1,16 @@ + + +# Vendored libbpf USDT header + +`usdt.h` is copied without modification from the version vendored by +libdatadog at commit `c3b0ee0a3f8a45e5a0cf153645ceffee6bbe7ec8`: + +https://raw.githubusercontent.com/DataDog/libdatadog/c3b0ee0a3f8a45e5a0cf153645ceffee6bbe7ec8/libdd-profiling-heap-sampler/vendor/usdt.h + +- Length: 25,031 bytes +- SHA-256: `5621bc8ae809d6e52f7833998194351f6b71c6fe83496b19e0d359757d18f8ee` +- Local modifications: none +- License: BSD-2-Clause (retained in the header) + +This is a compile-time-only header. The profiler does not link against libbpf +or libdatadog. diff --git a/ddprof-lib/src/main/cpp/vendor/libbpf/usdt.h b/ddprof-lib/src/main/cpp/vendor/libbpf/usdt.h new file mode 100644 index 0000000000..549d1f7748 --- /dev/null +++ b/ddprof-lib/src/main/cpp/vendor/libbpf/usdt.h @@ -0,0 +1,545 @@ +// SPDX-License-Identifier: BSD-2-Clause +/* + * This single-header library defines a collection of variadic macros for + * defining and triggering USDTs (User Statically-Defined Tracepoints): + * + * - For USDTs without associated semaphore: + * USDT(group, name, args...) + * + * - For USDTs with implicit (transparent to the user) semaphore: + * USDT_WITH_SEMA(group, name, args...) + * USDT_IS_ACTIVE(group, name) + * + * - For USDTs with explicit (user-defined and provided) semaphore: + * USDT_WITH_EXPLICIT_SEMA(sema, group, name, args...) + * USDT_SEMA_IS_ACTIVE(sema) + * + * all of which emit a NOP instruction into the instruction stream, and so + * have *zero* overhead for the surrounding code. USDTs are identified by + * a combination of `group` and `name` identifiers, which is used by external + * tracing tooling (tracers) for identifying exact USDTs of interest. + * + * USDTs can have an associated (2-byte) activity counter (USDT semaphore), + * automatically maintained by Linux kernel whenever any correctly written + * BPF-based tracer is attached to the USDT. This USDT semaphore can be used + * to check whether there is a need to do any extra data collection and + * processing for a given USDT (if necessary), and otherwise avoid extra work + * for a common case of USDT not being traced ("active"). + * + * See documentation for USDT_WITH_SEMA()/USDT_IS_ACTIVE() or + * USDT_WITH_EXPLICIT_SEMA()/USDT_SEMA_IS_ACTIVE() APIs below for details on + * working with USDTs with implicitly or explicitly associated + * USDT semaphores, respectively. + * + * There is also some additional data recorded into an auxiliary note + * section. The data in the note section describes the operands, in terms of + * size and location, used by tracing tooling to know where to find USDT + * arguments. Each location is encoded as an assembler operand string. + * Tracing tools (bpftrace and BPF-based tracers, systemtap, etc) insert + * breakpoints on top of the nop, and decode the location operand-strings, + * like an assembler, to find the values being passed. + * + * The operand strings are selected by the compiler for each operand. + * They are constrained by inline-assembler codes.The default is: + * + * #define USDT_ARG_CONSTRAINT nor + * + * This is a good default if the operands tend to be integral and + * moderate in number (smaller than number of registers). In other + * cases, the compiler may report "'asm' requires impossible reload" or + * similar. In this case, consider simplifying the macro call (fewer + * and simpler operands), reduce optimization, or override the default + * constraints string via: + * + * #define USDT_ARG_CONSTRAINT g + * #include + * + * For some historical description of USDT v3 format (the one used by this + * library and generally recognized and assumed by BPF-based tracing tools) + * see [0]. The more formal specification can be found at [1]. Additional + * argument constraints information can be found at [2]. + * + * Original SystemTap's sys/sdt.h implementation ([3]) was used as a base for + * this USDT library implementation. Current implementation differs *a lot* in + * terms of exposed user API and general usability, which was the main goal + * and focus of the reimplementation work. Nevertheless, underlying recorded + * USDT definitions are fully binary compatible and any USDT-based tooling + * should work equally well with USDTs defined by either SystemTap's or this + * library's USDT implementation. + * + * [0] https://ecos.sourceware.org/ml/systemtap/2010-q3/msg00145.html + * [1] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation + * [2] https://gcc.gnu.org/onlinedocs/gcc/Constraints.html + * [3] https://sourceware.org/git/?p=systemtap.git;a=blob;f=includes/sys/sdt.h + */ +#ifndef __USDT_H +#define __USDT_H + +/* + * Changelog: + * + * 0.1.0 + * ----- + * - Initial release + */ +#define USDT_MAJOR_VERSION 0 +#define USDT_MINOR_VERSION 1 +#define USDT_PATCH_VERSION 0 + +/* C++20 and C23 added __VA_OPT__ as a standard replacement for non-standard `##__VA_ARGS__` extension */ +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ > 201710L) || (defined(__cplusplus) && __cplusplus > 201703L) +#define __usdt_va_opt 1 +#define __usdt_va_args(...) __VA_OPT__(,) __VA_ARGS__ +#else +#define __usdt_va_args(...) , ##__VA_ARGS__ +#endif + +/* + * Trigger USDT with `group`:`name` identifier and pass through `args` as its + * arguments. Zero arguments are acceptable as well. No USDT semaphore is + * associated with this USDT. + * + * Such "semaphoreless" USDTs are commonly used when there is no extra data + * collection or processing needed to collect and prepare USDT arguments and + * they are just available in the surrounding code. USDT() macro will just + * record their locations in CPU registers or in memory for tracing tooling to + * be able to access them, if necessary. + */ +#ifdef __usdt_va_opt +#define USDT(group, name, ...) \ + __usdt_probe(group, name, __usdt_sema_none, 0 __VA_OPT__(,) __VA_ARGS__) +#else +#define USDT(group, name, ...) \ + __usdt_probe(group, name, __usdt_sema_none, 0, ##__VA_ARGS__) +#endif + +/* + * Trigger USDT with `group`:`name` identifier and pass through `args` as its + * arguments. Zero arguments are acceptable as well. USDT also get an + * implicitly-defined associated USDT semaphore, which will be "activated" by + * tracing tooling and can be used to check whether USDT is being actively + * observed. + * + * USDTs with semaphore are commonly used when there is a need to perform + * additional data collection and processing to prepare USDT arguments, which + * otherwise might not be necessary for the rest of application logic. In such + * case, USDT semaphore can be used to avoid unnecessary extra work. If USDT + * is not traced (which is presumed to be a common situation), the associated + * USDT semaphore is "inactive", and so there is no need to waste resources to + * prepare USDT arguments. Use USDT_IS_ACTIVE(group, name) to check whether + * USDT is "active". + * + * N.B. There is an inherent (albeit short) gap between checking whether USDT + * is active and triggering corresponding USDT, in which external tracer can + * be attached to an USDT and activate USDT semaphore after the activity check. + * If such a race occurs, tracers might miss one USDT execution. Tracers are + * expected to accommodate such possibility and this is expected to not be + * a problem for applications and tracers. + * + * N.B. Implicit USDT semaphore defined by USDT_WITH_SEMA() is contained + * within a single executable or shared library and is not shared outside + * them. I.e., if you use USDT_WITH_SEMA() with the same USDT group and name + * identifier across executable and shared library, it will work and won't + * conflict, per se, but will define independent USDT semaphores, one for each + * shared library/executable in which USDT_WITH_SEMA(group, name) is used. + * That is, if you attach to this USDT in one shared library (or executable), + * then only USDT semaphore within that shared library (or executable) will be + * updated by the kernel, while other libraries (or executable) will not see + * activated USDT semaphore. In short, it's best to use unique USDT group:name + * identifiers across different shared libraries (and, equivalently, between + * executable and shared library). This is advanced consideration and is + * rarely (if ever) seen in practice, but just to avoid surprises this is + * called out here. (Static libraries become a part of final executable, once + * linked by linker, so the above considerations don't apply to them.) + */ +#ifdef __usdt_va_opt +#define USDT_WITH_SEMA(group, name, ...) \ + __usdt_probe(group, name, \ + __usdt_sema_implicit, __usdt_sema_name(group, name) \ + __VA_OPT__(,) __VA_ARGS__) +#else +#define USDT_WITH_SEMA(group, name, ...) \ + __usdt_probe(group, name, \ + __usdt_sema_implicit, __usdt_sema_name(group, name), \ + ##__VA_ARGS__) +#endif + +struct usdt_sema { volatile unsigned short active; }; + +/* + * Check if USDT with `group`:`name` identifier is "active" (i.e., whether it + * is attached to by external tracing tooling and is actively observed). + * + * This macro can be used to decide whether any additional and potentially + * expensive data collection or processing should be done to pass extra + * information into the given USDT. It is assumed that USDT is triggered with + * USDT_WITH_SEMA() macro which will implicitly define associated USDT + * semaphore. (If one needs more control over USDT semaphore, see + * USDT_DEFINE_SEMA() and USDT_WITH_EXPLICIT_SEMA() macros below.) + * + * N.B. Such checks are necessarily racy and speculative. Between checking + * whether USDT is active and triggering the USDT itself, tracer can be + * detached with no notification. This race should be extremely rare and worst + * case should result in one-time wasted extra data collection and processing. + */ +#define USDT_IS_ACTIVE(group, name) ({ \ + extern struct usdt_sema __usdt_sema_name(group, name) \ + __usdt_asm_name(__usdt_sema_name(group, name)); \ + __usdt_sema_implicit(__usdt_sema_name(group, name)); \ + __usdt_sema_name(group, name).active > 0; \ +}) + +/* + * APIs for working with user-defined explicit USDT semaphores. + * + * This is a less commonly used advanced API for use cases in which user needs + * an explicit control over (potentially shared across multiple USDTs) USDT + * semaphore instance. This can be used when there is a group of logically + * related USDTs that all need extra data collection and processing whenever + * any of a family of related USDTs are "activated" (i.e., traced). In such + * a case, all such related USDTs will be associated with the same shared USDT + * semaphore defined with USDT_DEFINE_SEMA() and the USDTs themselves will be + * triggered with USDT_WITH_EXPLICIT_SEMA() macros, taking an explicit extra + * USDT semaphore identifier as an extra parameter. + */ + +/** + * Underlying C global variable name for user-defined USDT semaphore with + * `sema` identifier. Could be useful for debugging, but normally shouldn't be + * used explicitly. + */ +#define USDT_SEMA(sema) __usdt_sema_##sema + +/* + * Define storage for user-defined USDT semaphore `sema`. + * + * Should be used only once in non-header source file to let compiler allocate + * space for the semaphore variable. Just like with any other global variable. + * + * This macro can be used anywhere where global variable declaration is + * allowed. Just like with global variable definitions, there should be only + * one definition of user-defined USDT semaphore with given `sema` identifier, + * otherwise compiler or linker will complain about duplicate variable + * definition. + * + * For C++, it is allowed to use USDT_DEFINE_SEMA() both in global namespace + * and inside namespaces (including nested namespaces). Just make sure that + * USDT_DECLARE_SEMA() is placed within the namespace where this semaphore is + * referenced, or any of its parent namespaces, so the C++ language-level + * identifier is visible to the code that needs to reference the semaphore. + * At the lowest layer, USDT semaphores have global naming and visibility + * (they have a corresponding `__usdt_sema_` symbol, which can be linked + * against from C or C++ code, if necessary). To keep it simple, putting + * USDT_DECLARE_SEMA() declarations into global namespaces is the simplest + * no-brainer solution. All these aspects are irrelevant for plain C, because + * C doesn't have namespaces and everything is always in the global namespace. + * + * N.B. Due to USDT metadata being recorded in non-allocatable ELF note + * section, it has limitations when it comes to relocations, which, in + * practice, means that it's not possible to correctly share USDT semaphores + * between main executable and shared libraries, or even between multiple + * shared libraries. USDT semaphore has to be contained to individual shared + * library or executable to avoid unpleasant surprises with half-working USDT + * semaphores. We enforce this by marking semaphore ELF symbols as having + * a hidden visibility. This is quite an advanced use case and consideration + * and for most users this should have no consequences whatsoever. + */ +#define USDT_DEFINE_SEMA(sema) \ + struct usdt_sema __usdt_sema_sec USDT_SEMA(sema) \ + __usdt_asm_name(USDT_SEMA(sema)) \ + __attribute__((visibility("hidden"))) = { 0 } + +/* + * Declare extern reference to user-defined USDT semaphore `sema`. + * + * Refers to a variable defined in another compilation unit by + * USDT_DEFINE_SEMA() and allows to use the same USDT semaphore across + * multiple compilation units (i.e., .c and .cpp files). + * + * See USDT_DEFINE_SEMA() notes above for C++ language usage peculiarities. + */ +#define USDT_DECLARE_SEMA(sema) \ + extern struct usdt_sema USDT_SEMA(sema) __usdt_asm_name(USDT_SEMA(sema)) + +/* + * Check if user-defined USDT semaphore `sema` is "active" (i.e., whether it + * is attached to by external tracing tooling and is actively observed). + * + * This macro can be used to decide whether any additional and potentially + * expensive data collection or processing should be done to pass extra + * information into USDT(s) associated with USDT semaphore `sema`. + * + * N.B. Such checks are necessarily racy. Between checking the state of USDT + * semaphore and triggering associated USDT(s), the active tracer might attach + * or detach. This race should be extremely rare and worst case should result + * in one-time missed USDT event or wasted extra data collection and + * processing. USDT-using tracers should be written with this in mind and is + * not a concern of the application defining USDTs with associated semaphore. + */ +#define USDT_SEMA_IS_ACTIVE(sema) (USDT_SEMA(sema).active > 0) + +/* + * Invoke USDT specified by `group` and `name` identifiers and associate + * explicitly user-defined semaphore `sema` with it. Pass through `args` as + * USDT arguments. `args` are optional and zero arguments are acceptable. + * + * Semaphore is defined with the help of USDT_DEFINE_SEMA() macro and can be + * checked whether active with USDT_SEMA_IS_ACTIVE(). + */ +#ifdef __usdt_va_opt +#define USDT_WITH_EXPLICIT_SEMA(sema, group, name, ...) \ + __usdt_probe(group, name, __usdt_sema_explicit, USDT_SEMA(sema), ##__VA_ARGS__) +#else +#define USDT_WITH_EXPLICIT_SEMA(sema, group, name, ...) \ + __usdt_probe(group, name, __usdt_sema_explicit, USDT_SEMA(sema) __VA_OPT__(,) __VA_ARGS__) +#endif + +/* + * Adjustable implementation aspects + */ +#ifndef USDT_ARG_CONSTRAINT +#if defined __powerpc__ +#define USDT_ARG_CONSTRAINT nZr +#elif defined __arm__ +#define USDT_ARG_CONSTRAINT g +#elif defined __loongarch__ +#define USDT_ARG_CONSTRAINT nmr +#else +#define USDT_ARG_CONSTRAINT nor +#endif +#endif /* USDT_ARG_CONSTRAINT */ + +#ifndef USDT_NOP +#if defined(__ia64__) || defined(__s390__) || defined(__s390x__) +#define USDT_NOP nop 0 +#else +#define USDT_NOP nop +#endif +#endif /* USDT_NOP */ + +/* + * Implementation details + */ +/* USDT name for implicitly-defined USDT semaphore, derived from group:name */ +#define __usdt_sema_name(group, name) __usdt_sema_##group##__##name +/* ELF section into which USDT semaphores are put */ +#define __usdt_sema_sec __attribute__((section(".probes"))) + +#define __usdt_concat(a, b) a ## b +#define __usdt_apply(fn, n) __usdt_concat(fn, n) + +#ifndef __usdt_nth +#define __usdt_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, N, ...) N +#endif + +#ifndef __usdt_narg +#ifdef __usdt_va_opt +#define __usdt_narg(...) __usdt_nth(_ __VA_OPT__(,) __VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#else +#define __usdt_narg(...) __usdt_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#endif +#endif /* __usdt_narg */ + +#define __usdt_hash # +#define __usdt_str_(x) #x +#define __usdt_str(x) __usdt_str_(x) + +#ifndef __usdt_asm_name +#define __usdt_asm_name(name) __asm__(__usdt_str(name)) +#endif + +#define __usdt_asm0() "\n" +#define __usdt_asm1(x) __usdt_str(x) "\n" +#define __usdt_asm2(x, ...) __usdt_str(x) "," __usdt_asm1(__VA_ARGS__) +#define __usdt_asm3(x, ...) __usdt_str(x) "," __usdt_asm2(__VA_ARGS__) +#define __usdt_asm4(x, ...) __usdt_str(x) "," __usdt_asm3(__VA_ARGS__) +#define __usdt_asm5(x, ...) __usdt_str(x) "," __usdt_asm4(__VA_ARGS__) +#define __usdt_asm6(x, ...) __usdt_str(x) "," __usdt_asm5(__VA_ARGS__) +#define __usdt_asm7(x, ...) __usdt_str(x) "," __usdt_asm6(__VA_ARGS__) +#define __usdt_asm8(x, ...) __usdt_str(x) "," __usdt_asm7(__VA_ARGS__) +#define __usdt_asm9(x, ...) __usdt_str(x) "," __usdt_asm8(__VA_ARGS__) +#define __usdt_asm10(x, ...) __usdt_str(x) "," __usdt_asm9(__VA_ARGS__) +#define __usdt_asm11(x, ...) __usdt_str(x) "," __usdt_asm10(__VA_ARGS__) +#define __usdt_asm12(x, ...) __usdt_str(x) "," __usdt_asm11(__VA_ARGS__) +#define __usdt_asm(...) __usdt_apply(__usdt_asm, __usdt_narg(__VA_ARGS__))(__VA_ARGS__) + +#ifdef __LP64__ +#define __usdt_asm_addr .8byte +#else +#define __usdt_asm_addr .4byte +#endif + +#define __usdt_asm_strz_(x) __usdt_asm1(.asciz #x) +#define __usdt_asm_strz(x) __usdt_asm_strz_(x) +#define __usdt_asm_str_(x) __usdt_asm1(.ascii #x) +#define __usdt_asm_str(x) __usdt_asm_str_(x) + +/* "semaphoreless" USDT case */ +#ifndef __usdt_sema_none +#define __usdt_sema_none(sema) +#endif + +/* implicitly defined __usdt_sema__group__name semaphore (using weak symbols) */ +#ifndef __usdt_sema_implicit +#define __usdt_sema_implicit(sema) \ + __asm__ __volatile__ ( \ + __usdt_asm1(.ifndef sema) \ + __usdt_asm3( .pushsection .probes, "aw", "progbits") \ + __usdt_asm1( .weak sema) \ + __usdt_asm1( .hidden sema) \ + __usdt_asm1( .align 2) \ + __usdt_asm1(sema:) \ + __usdt_asm1( .zero 2) \ + __usdt_asm2( .type sema, @object) \ + __usdt_asm2( .size sema, 2) \ + __usdt_asm1( .popsection) \ + __usdt_asm1(.endif) \ + ); +#endif + +/* externally defined semaphore using USDT_DEFINE_SEMA() and passed explicitly by user */ +#ifndef __usdt_sema_explicit +#define __usdt_sema_explicit(sema) \ + __asm__ __volatile__ ("" :: "m" (sema)); +#endif + +/* main USDT definition (nop and .note.stapsdt metadata) */ +#define __usdt_probe(group, name, sema_def, sema, ...) do { \ + sema_def(sema) \ + __asm__ __volatile__ ( \ + __usdt_asm( 990: USDT_NOP) \ + __usdt_asm3( .pushsection .note.stapsdt, "", "note") \ + __usdt_asm1( .balign 4) \ + __usdt_asm3( .4byte 992f-991f,994f-993f,3) \ + __usdt_asm1(991: .asciz "stapsdt") \ + __usdt_asm1(992: .balign 4) \ + __usdt_asm1(993: __usdt_asm_addr 990b) \ + __usdt_asm1( __usdt_asm_addr _.stapsdt.base) \ + __usdt_asm1( __usdt_asm_addr sema) \ + __usdt_asm_strz(group) \ + __usdt_asm_strz(name) \ + __usdt_asm_args(__VA_ARGS__) \ + __usdt_asm1( .ascii "\0") \ + __usdt_asm1(994: .balign 4) \ + __usdt_asm1( .popsection) \ + __usdt_asm1(.ifndef _.stapsdt.base) \ + __usdt_asm5( .pushsection .stapsdt.base,"aG","progbits",.stapsdt.base,comdat)\ + __usdt_asm1( .weak _.stapsdt.base) \ + __usdt_asm1( .hidden _.stapsdt.base) \ + __usdt_asm1(_.stapsdt.base:) \ + __usdt_asm1( .space 1) \ + __usdt_asm2( .size _.stapsdt.base, 1) \ + __usdt_asm1( .popsection) \ + __usdt_asm1(.endif) \ + :: __usdt_asm_ops(__VA_ARGS__) \ + ); \ +} while (0) + +/* + * NB: gdb PR24541 highlighted an unspecified corner of the sdt.h + * operand note format. + * + * The named register may be a longer or shorter (!) alias for the + * storage where the value in question is found. For example, on + * i386, 64-bit value may be put in register pairs, and a register + * name stored would identify just one of them. Previously, gcc was + * asked to emit the %w[id] (16-bit alias of some registers holding + * operands), even when a wider 32-bit value was used. + * + * Bottom line: the byte-width given before the @ sign governs. If + * there is a mismatch between that width and that of the named + * register, then a sys/sdt.h note consumer may need to employ + * architecture-specific heuristics to figure out where the compiler + * has actually put the complete value. + */ +#if defined(__powerpc__) || defined(__powerpc64__) +#define __usdt_argref(id) %I[id]%[id] +#elif defined(__i386__) +#define __usdt_argref(id) %k[id] /* gcc.gnu.org/PR80115 sourceware.org/PR24541 */ +#else +#define __usdt_argref(id) %[id] +#endif + +#define __usdt_asm_arg(n) __usdt_asm_str(%c[__usdt_asz##n]) \ + __usdt_asm1(.ascii "@") \ + __usdt_asm_str(__usdt_argref(__usdt_aval##n)) + +#define __usdt_asm_args0 /* no arguments */ +#define __usdt_asm_args1 __usdt_asm_arg(1) +#define __usdt_asm_args2 __usdt_asm_args1 __usdt_asm1(.ascii " ") __usdt_asm_arg(2) +#define __usdt_asm_args3 __usdt_asm_args2 __usdt_asm1(.ascii " ") __usdt_asm_arg(3) +#define __usdt_asm_args4 __usdt_asm_args3 __usdt_asm1(.ascii " ") __usdt_asm_arg(4) +#define __usdt_asm_args5 __usdt_asm_args4 __usdt_asm1(.ascii " ") __usdt_asm_arg(5) +#define __usdt_asm_args6 __usdt_asm_args5 __usdt_asm1(.ascii " ") __usdt_asm_arg(6) +#define __usdt_asm_args7 __usdt_asm_args6 __usdt_asm1(.ascii " ") __usdt_asm_arg(7) +#define __usdt_asm_args8 __usdt_asm_args7 __usdt_asm1(.ascii " ") __usdt_asm_arg(8) +#define __usdt_asm_args9 __usdt_asm_args8 __usdt_asm1(.ascii " ") __usdt_asm_arg(9) +#define __usdt_asm_args10 __usdt_asm_args9 __usdt_asm1(.ascii " ") __usdt_asm_arg(10) +#define __usdt_asm_args11 __usdt_asm_args10 __usdt_asm1(.ascii " ") __usdt_asm_arg(11) +#define __usdt_asm_args12 __usdt_asm_args11 __usdt_asm1(.ascii " ") __usdt_asm_arg(12) +#define __usdt_asm_args(...) __usdt_apply(__usdt_asm_args, __usdt_narg(__VA_ARGS__)) + +#define __usdt_is_arr(x) (__builtin_classify_type(x) == 14 || __builtin_classify_type(x) == 5) +#define __usdt_arg_size(x) (__usdt_is_arr(x) ? sizeof(void *) : sizeof(x)) + +/* + * We can't use __builtin_choose_expr() in C++, so fall back to table-based + * signedness determination for known types, utilizing templates magic. + */ +#ifdef __cplusplus + +#define __usdt_is_signed(x) (!__usdt_is_arr(x) && __usdt_t<__typeof(x)>::is_signed) + +#include + +template struct __usdt_t { static const bool is_signed = false; }; +template struct __usdt_t : public __usdt_t {}; +template struct __usdt_t : public __usdt_t {}; + +#define __usdt_def_signed(T) \ +template<> struct __usdt_t { static const bool is_signed = true; }; \ +template<> struct __usdt_t { static const bool is_signed = true; }; \ +template<> struct __usdt_t { static const bool is_signed = true; }; \ +template<> struct __usdt_t { static const bool is_signed = true; } +#define __usdt_maybe_signed(T) \ +template<> struct __usdt_t { static const bool is_signed = (T)-1 < (T)1; }; \ +template<> struct __usdt_t { static const bool is_signed = (T)-1 < (T)1; }; \ +template<> struct __usdt_t { static const bool is_signed = (T)-1 < (T)1; }; \ +template<> struct __usdt_t { static const bool is_signed = (T)-1 < (T)1; } + +__usdt_def_signed(signed char); +__usdt_def_signed(short); +__usdt_def_signed(int); +__usdt_def_signed(long); +__usdt_def_signed(long long); +__usdt_maybe_signed(char); +__usdt_maybe_signed(wchar_t); + +#else /* !__cplusplus */ + +#define __usdt_is_inttype(x) (__builtin_classify_type(x) >= 1 && __builtin_classify_type(x) <= 4) +#define __usdt_inttype(x) __typeof(__builtin_choose_expr(__usdt_is_inttype(x), (x), 0U)) +#define __usdt_is_signed(x) ((__usdt_inttype(x))-1 < (__usdt_inttype(x))1) + +#endif /* __cplusplus */ + +#define __usdt_asm_op(n, x) \ + [__usdt_asz##n] "n" ((__usdt_is_signed(x) ? (int)-1 : 1) * (int)__usdt_arg_size(x)), \ + [__usdt_aval##n] __usdt_str(USDT_ARG_CONSTRAINT)(x) + +#define __usdt_asm_ops0() [__usdt_dummy] "g" (0) +#define __usdt_asm_ops1(x) __usdt_asm_op(1, x) +#define __usdt_asm_ops2(a,x) __usdt_asm_ops1(a), __usdt_asm_op(2, x) +#define __usdt_asm_ops3(a,b,x) __usdt_asm_ops2(a,b), __usdt_asm_op(3, x) +#define __usdt_asm_ops4(a,b,c,x) __usdt_asm_ops3(a,b,c), __usdt_asm_op(4, x) +#define __usdt_asm_ops5(a,b,c,d,x) __usdt_asm_ops4(a,b,c,d), __usdt_asm_op(5, x) +#define __usdt_asm_ops6(a,b,c,d,e,x) __usdt_asm_ops5(a,b,c,d,e), __usdt_asm_op(6, x) +#define __usdt_asm_ops7(a,b,c,d,e,f,x) __usdt_asm_ops6(a,b,c,d,e,f), __usdt_asm_op(7, x) +#define __usdt_asm_ops8(a,b,c,d,e,f,g,x) __usdt_asm_ops7(a,b,c,d,e,f,g), __usdt_asm_op(8, x) +#define __usdt_asm_ops9(a,b,c,d,e,f,g,h,x) __usdt_asm_ops8(a,b,c,d,e,f,g,h), __usdt_asm_op(9, x) +#define __usdt_asm_ops10(a,b,c,d,e,f,g,h,i,x) __usdt_asm_ops9(a,b,c,d,e,f,g,h,i), __usdt_asm_op(10, x) +#define __usdt_asm_ops11(a,b,c,d,e,f,g,h,i,j,x) __usdt_asm_ops10(a,b,c,d,e,f,g,h,i,j), __usdt_asm_op(11, x) +#define __usdt_asm_ops12(a,b,c,d,e,f,g,h,i,j,k,x) __usdt_asm_ops11(a,b,c,d,e,f,g,h,i,j,k), __usdt_asm_op(12, x) +#define __usdt_asm_ops(...) __usdt_apply(__usdt_asm_ops, __usdt_narg(__VA_ARGS__))(__VA_ARGS__) + +#endif /* __USDT_H */ diff --git a/ddprof-lib/src/test/cpp/arguments_ut.cpp b/ddprof-lib/src/test/cpp/arguments_ut.cpp index 8ecefc13fc..50c90f6001 100644 --- a/ddprof-lib/src/test/cpp/arguments_ut.cpp +++ b/ddprof-lib/src/test/cpp/arguments_ut.cpp @@ -6,6 +6,7 @@ #include #include #include "arguments.h" +#include "mallocTracer.h" #include "../../main/cpp/gtest_crash_handler.h" static constexpr char ARGUMENTS_TEST_NAME[] = "ArgumentsTest"; @@ -28,6 +29,75 @@ class ArgumentsTest : public ::testing::Test { void TearDown() override {} }; +TEST_F(ArgumentsTest, NativeMemoryOutputDefaultsToJfrAndIsNotExplicit) { + Arguments args; + EXPECT_EQ(args._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_FALSE(args._nativemem_output_set); + + ASSERT_FALSE(args.parse("nativemem=512")); + EXPECT_EQ(args._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_FALSE(args._nativemem_output_set); +} + +TEST_F(ArgumentsTest, NativeMemoryOutputParsesAllValuesCaseInsensitively) { + Arguments jfr; + ASSERT_FALSE(jfr.parse("nativemem=512,nativememoutput=JFR")); + EXPECT_EQ(jfr._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_TRUE(jfr._nativemem_output_set); + + Arguments usdt; + ASSERT_FALSE(usdt.parse("nativememoutput=UsDt,nativemem=512")); + EXPECT_EQ(usdt._nativemem_output, NATIVE_MEMORY_USDT); + EXPECT_TRUE(usdt._nativemem_output_set); + + Arguments both; + ASSERT_FALSE(both.parse("nativemem=512,nativememoutput=BoTh")); + EXPECT_EQ(both._nativemem_output, NATIVE_MEMORY_BOTH); + EXPECT_TRUE(both._nativemem_output_set); +} + +TEST_F(ArgumentsTest, NativeMemoryOutputRejectsMissingEmptyAndUnknownValues) { + Arguments missing; + EXPECT_TRUE(missing.parse("nativememoutput")); + + Arguments empty; + EXPECT_TRUE(empty.parse("nativememoutput=")); + + Arguments unknown; + EXPECT_TRUE(unknown.parse("nativememoutput=jfr-and-usdt")); +} + +TEST_F(ArgumentsTest, NativeMemoryOutputRequiresNativeMemoryRegardlessOfOrder) { + MallocTracer tracer; + + Arguments output_only; + ASSERT_FALSE(output_only.parse("nativememoutput=jfr")); + EXPECT_TRUE(tracer.check(output_only)); + + Arguments output_first; + ASSERT_FALSE(output_first.parse("nativememoutput=usdt,nativemem=512")); +#if defined(__linux__) + EXPECT_FALSE(tracer.check(output_first)); +#else + EXPECT_TRUE(tracer.check(output_first)); +#endif + + Arguments memory_first; + ASSERT_FALSE(memory_first.parse("nativemem=512,nativememoutput=both")); +#if defined(__linux__) + EXPECT_FALSE(tracer.check(memory_first)); +#else + EXPECT_TRUE(tracer.check(memory_first)); +#endif +} + +TEST_F(ArgumentsTest, ExplicitJfrRemainsSupportedOnEveryPlatform) { + MallocTracer tracer; + Arguments args; + ASSERT_FALSE(args.parse("nativemem=512,nativememoutput=jfr")); + EXPECT_FALSE(tracer.check(args)); +} + // hops/budget/framecap are ceiling-clamped (MAX_REFERENCE_CHAINS_HOP_CAP/ // _BUDGET/_FRONTIER_CAP, arguments.h) as well as floored at 1 - an operator // typo (an extra digit) must not flow straight into a loop bound or diff --git a/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp b/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp new file mode 100644 index 0000000000..0f14af772b --- /dev/null +++ b/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp @@ -0,0 +1,251 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include +#include + +#include "mallocTracer.h" +#include "nativeAllocationUsdt.h" +#include "threadLocalData.inline.h" +#include "threadLocalDataPool.h" + +namespace { + +int jfr_count; +int active_count; +int emit_count; +uint16_t active_value; +void* emitted_address; +uint64_t emitted_size; +uint64_t emitted_weight; +bool sample_result; +bool critical_result; +ProfiledThread* acquire_result; +const char* stages[16]; +size_t stage_count; + +ProfiledThread* acquireHook() { + return acquire_result; +} + +bool criticalHook(bool entered) { + return entered && critical_result; +} + +bool sampleHook(size_t) { + return sample_result; +} + +void jfrHook(void*, size_t, int, MallocEvent*) { + jfr_count++; +} + +bool activeHook() { + active_count++; + return active_value != 0; +} + +void emitHook(void* address, uint64_t size, uint64_t weight) { + emit_count++; + emitted_address = address; + emitted_size = size; + emitted_weight = weight; +} + +void stageHook(const char* stage) { + ASSERT_LT(stage_count, sizeof(stages) / sizeof(stages[0])); + stages[stage_count++] = stage; +} + +class MallocTracerUsdtTest : public ::testing::Test { +protected: + static void SetUpTestSuite() { + if (!ThreadLocalDataPool::isInitialized()) { + ThreadLocalDataPool::initialize(); + } + ASSERT_NE(nullptr, ProfiledThread::initCurrentThreadSignalSafe()); + } + + void SetUp() override { + jfr_count = 0; + active_count = 0; + emit_count = 0; + active_value = 0; + emitted_address = nullptr; + emitted_size = 0; + emitted_weight = 0; + sample_result = true; + critical_result = true; + acquire_result = ProfiledThread::current(); + stage_count = 0; + MallocTracer::configureForTest(NATIVE_MEMORY_JFR, 512, true); + MallocTracer::setTestHooks(acquireHook, criticalHook, sampleHook, + jfrHook, activeHook, emitHook, stageHook); + } + + void TearDown() override { + MallocTracer::resetTestHooks(); + } + + void expectStages(std::initializer_list expected) { + ASSERT_EQ(expected.size(), stage_count); + size_t index = 0; + for (const char* stage : expected) { + EXPECT_STREQ(stage, stages[index++]); + } + } +}; + +TEST_F(MallocTracerUsdtTest, WeightUsesWeightedBytesContract) { + EXPECT_EQ(0u, nativeAllocationUsdtWeight(0, 512)); + EXPECT_EQ(1u, nativeAllocationUsdtWeight(1, 0)); + EXPECT_EQ(17u, nativeAllocationUsdtWeight(17, 1)); + EXPECT_EQ(513u, nativeAllocationUsdtWeight(1, 512)); + EXPECT_EQ(810u, nativeAllocationUsdtWeight(512, 512)); + EXPECT_EQ(1184u, nativeAllocationUsdtWeight(1024, 512)); + EXPECT_EQ(4097u, nativeAllocationUsdtWeight(4096, 512)); + EXPECT_EQ((uint64_t)INT64_MAX, + nativeAllocationUsdtWeight((size_t)INT64_MAX, UINT64_MAX)); +} + +TEST_F(MallocTracerUsdtTest, JfrModeNeverQueriesOrEmitsUsdt) { + active_value = 1; + MallocTracer::recordMalloc(reinterpret_cast(0x1234), 512, 7); + + EXPECT_EQ(1, jfr_count); + EXPECT_EQ(0, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"jfr", "accounting"}); +} + +TEST_F(MallocTracerUsdtTest, UsdtModeSkipsJfrAndEmitsWeightedPayload) { + active_value = 2; + void* address = reinterpret_cast(0x1234); + MallocTracer::configureForTest(NATIVE_MEMORY_USDT, 512, true); + MallocTracer::recordMalloc(address, 512, 7); + + EXPECT_EQ(0, jfr_count); + EXPECT_EQ(1, active_count); + EXPECT_EQ(1, emit_count); + EXPECT_EQ(address, emitted_address); + EXPECT_EQ(512u, emitted_size); + EXPECT_EQ(810u, emitted_weight); + expectStages({"usdt", "accounting"}); +} + +TEST_F(MallocTracerUsdtTest, BothModeWritesJfrBeforeUsdtThenAccountsOnce) { + active_value = 1; + MallocTracer::configureForTest(NATIVE_MEMORY_BOTH, 512, true); + MallocTracer::recordMalloc(reinterpret_cast(0x1234), 512, 7); + + EXPECT_EQ(1, jfr_count); + EXPECT_EQ(1, active_count); + EXPECT_EQ(1, emit_count); + expectStages({"jfr", "usdt", "accounting"}); +} + +TEST_F(MallocTracerUsdtTest, EveryNonzeroSemaphoreValueIsActive) { + MallocTracer::configureForTest(NATIVE_MEMORY_USDT, 1, true); + for (uint16_t value : {uint16_t(0), uint16_t(1), uint16_t(2), uint16_t(UINT16_MAX)}) { + active_value = value; + int before = emit_count; + MallocTracer::recordMalloc(reinterpret_cast(0x1234), 8, 7); + EXPECT_EQ(before + (value == 0 ? 0 : 1), emit_count); + } + EXPECT_EQ(4, active_count); +} + +#if defined(__linux__) +TEST_F(MallocTracerUsdtTest, SemaphoreUsesTheKernelReferenceCountSemantics) { + for (uint16_t value : {uint16_t(0), uint16_t(1), uint16_t(2), UINT16_MAX}) { + setNativeAllocationUsdtSemaphoreForTest(value); + EXPECT_EQ(value != 0, isNativeAllocationUsdtEnabled()); + } + setNativeAllocationUsdtSemaphoreForTest(0); +} +#endif + +TEST_F(MallocTracerUsdtTest, AttachDetachDoesNotResetAcceptedSampleAccounting) { + MallocTracer::configureForTest(NATIVE_MEMORY_USDT, 1, true); + active_value = 0; + MallocTracer::recordMalloc(reinterpret_cast(0x1000), 8, 7); + active_value = 1; + MallocTracer::recordMalloc(reinterpret_cast(0x2000), 8, 7); + active_value = 0; + MallocTracer::recordMalloc(reinterpret_cast(0x3000), 8, 7); + + EXPECT_EQ(3, active_count); + EXPECT_EQ(1, emit_count); + size_t accounting_count = 0; + for (size_t i = 0; i < stage_count; i++) { + if (strcmp(stages[i], "accounting") == 0) accounting_count++; + } + EXPECT_EQ(3u, accounting_count); +} + +#if SIZE_MAX > INT64_MAX +TEST_F(MallocTracerUsdtTest, OversizedAllocationStillUsesJfrAndAccountingInBothMode) { + active_value = 1; + MallocTracer::configureForTest(NATIVE_MEMORY_BOTH, 512, true); + MallocTracer::recordMalloc(reinterpret_cast(0x1234), + (size_t)INT64_MAX + 1, 7); + + EXPECT_EQ(1, jfr_count); + EXPECT_EQ(0, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"jfr", "accounting"}); +} +#endif + +TEST_F(MallocTracerUsdtTest, MaybeRecordPreservesGuardAndOutputOrder) { + MallocTracer::maybeRecord(reinterpret_cast(0x1234), 512); + expectStages({"acquire", "critical-section", "sampling", "jfr", "accounting"}); +} + +TEST_F(MallocTracerUsdtTest, AcquisitionFailureStopsBeforeGuardAndSampling) { + acquire_result = nullptr; + MallocTracer::maybeRecord(reinterpret_cast(0x1234), 512); + + EXPECT_EQ(0, jfr_count); + EXPECT_EQ(0, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"acquire"}); +} + +TEST_F(MallocTracerUsdtTest, CriticalSectionRejectionStopsBeforeSampling) { + critical_result = false; + MallocTracer::maybeRecord(reinterpret_cast(0x1234), 512); + + EXPECT_EQ(0, jfr_count); + EXPECT_EQ(0, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"acquire", "critical-section"}); +} + +TEST_F(MallocTracerUsdtTest, RejectedSampleHasNoOutputOrAccounting) { + sample_result = false; + MallocTracer::maybeRecord(reinterpret_cast(0x1234), 512); + + EXPECT_EQ(0, jfr_count); + EXPECT_EQ(0, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"acquire", "critical-section", "sampling"}); +} + +TEST_F(MallocTracerUsdtTest, InactiveUsdtSkipsWeightAndEmissionButAccounts) { + MallocTracer::configureForTest(NATIVE_MEMORY_USDT, 512, true); + active_value = 0; + MallocTracer::recordMalloc(reinterpret_cast(0x1234), 512, 7); + + EXPECT_EQ(0, jfr_count); + EXPECT_EQ(1, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"accounting"}); +} + +} // namespace diff --git a/doc/architecture/NativeMemoryProfiling.md b/doc/architecture/NativeMemoryProfiling.md index d2f68b4525..2b0800f175 100644 --- a/doc/architecture/NativeMemoryProfiling.md +++ b/doc/architecture/NativeMemoryProfiling.md @@ -1,3 +1,5 @@ + + # Native Memory Allocation Profiling ## Overview @@ -6,15 +8,26 @@ The native memory profiler tracks heap allocations made through the C standard library (`malloc`, `calloc`, `realloc`, `posix_memalign`, `aligned_alloc`). It instruments these functions at the GOT (Global Offset Table) level so that every intercepted call is accounted for without modifying application source code or -requiring a custom allocator. The `free` function is also hooked (to forward calls -correctly through the GOT) but free events are not recorded. +requiring a custom allocator. Free events are not intercepted or recorded. -Sampled allocation events carry a full Java + native stack trace and are emitted as -`datadog.NativeMemoryAllocation` JFR events. +By default, sampled allocation events carry a full Java + native stack trace and +are emitted as `datadog.NativeMemoryAllocation` JFR events. Linux x86-64 and +AArch64 builds can instead expose sampled allocations through the semaphored +`ddheap:alloc(address, size, weighted_bytes)` USDT probe for an external eBPF +profiler. The feature is activated by passing `nativemem=` to the profiler, where `` is the byte-sampling interval (e.g. `nativemem=524288` samples roughly one event per 512 KiB allocated). Passing `nativemem=0` records every allocation. +`nativememoutput=jfr|usdt|both` selects the destination; `jfr` is the default. +The output option does not activate profiling by itself and `usdt`/`both` are +rejected outside Linux. + +USDT output is allocation-only. It deliberately does not add a free probe or +retain live-heap state. `weighted_bytes` is the allocation size divided by its +sampling probability and is capped at `INT64_MAX` for the current consumer's +signed profile values. USDT-only mode continues the sampler while no consumer is +attached, but skips both the JFR stack walk and probe payload calculation. --- @@ -22,10 +35,10 @@ one event per 512 KiB allocated). Passing `nativemem=0` records every allocation ``` Application code - │ malloc() / calloc() / realloc() / free() / … + │ malloc() / calloc() / realloc() / … ▼ ┌─────────────┐ GOT patch ┌──────────────────────────┐ - │ libc / musl│ ◄────────── │ malloc_hook / free_hook │ mallocTracer.cpp + │ libc / musl│ ◄────────── │ malloc_hook / calloc_hook│ mallocTracer.cpp └─────────────┘ │ calloc_hook / … │ └────────────┬─────────────┘ │ recordMalloc @@ -56,7 +69,7 @@ restart) and works for libraries loaded at any time. `codeCache.h` defines an `ImportId` enum with one entry per hooked symbol: ``` -im_malloc, im_calloc, im_realloc, im_free, im_posix_memalign, im_aligned_alloc +im_malloc, im_calloc, im_realloc, im_posix_memalign, im_aligned_alloc ``` `CodeCache::patchImport(ImportId, void*)` walks the library's PLT/GOT and overwrites @@ -71,7 +84,6 @@ Each hook calls the saved original function first, then records the event: | `malloc_hook(size)` | `_orig_malloc(size)` | `recordMalloc(ret, size)` if `ret != NULL && size != 0` | | `calloc_hook(num, size)` | `_orig_calloc(num, size)` | `recordMalloc(ret, total)` if `ret != NULL && num != 0 && size != 0` (total = num×size, clamped to `SIZE_MAX` on overflow) | | `realloc_hook(addr, size)` | `_orig_realloc(addr, size)` | `recordMalloc(ret, size)` if `ret != NULL && size > 0` | -| `free_hook(addr)` | `_orig_free(addr)` | — (forwards only) | | `posix_memalign_hook(…)` | `_orig_posix_memalign(…)` | `recordMalloc(*memptr, size)` if `ret == 0 && memptr != NULL && *memptr != NULL && size != 0` | | `aligned_alloc_hook(align, size)` | `_orig_aligned_alloc(align, size)` | `recordMalloc(ret, size)` if `ret != NULL && size != 0` | From b7e70ebbd0378ae0730e38b52c95391bcfbfd994 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 20 Oct 2026 12:48:39 +0200 Subject: [PATCH 2/5] Verify packaged native allocation USDT probe --- ddprof-lib/build.gradle.kts | 22 +++++++ utils/verify-native-allocation-usdt.sh | 81 ++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100755 utils/verify-native-allocation-usdt.sh diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts index ff194cccf6..cf1387a5ab 100644 --- a/ddprof-lib/build.gradle.kts +++ b/ddprof-lib/build.gradle.kts @@ -17,6 +17,7 @@ import com.datadoghq.native.model.Platform import com.datadoghq.native.util.PlatformUtils import org.gradle.api.publish.maven.tasks.AbstractPublishToMaven +import org.gradle.api.tasks.Exec import org.gradle.api.tasks.VerificationTask plugins { @@ -173,6 +174,27 @@ afterEvaluate { archiveVersion.set(componentVersion) } + if (name == "release") { + tasks.register("verifyNativeAllocationUsdtRelease", Exec::class) { + group = "verification" + description = "Verify the packaged Linux release USDT allocation probe" + dependsOn(assembleJarTask) + + val verifier = rootProject.layout.projectDirectory.file("utils/verify-native-allocation-usdt.sh") + val releaseJar = assembleJarTask.flatMap { it.archiveFile } + inputs.file(verifier) + inputs.file(releaseJar) + + onlyIf { + PlatformUtils.currentPlatform == Platform.LINUX && + !PlatformUtils.isNativeSkipped(project) + } + doFirst { + commandLine("bash", verifier.asFile.absolutePath, releaseJar.get().asFile.absolutePath) + } + } + } + // Create consumable configuration for inter-project dependencies // This allows other projects to depend on specific build configurations configurations.create(name) { diff --git a/utils/verify-native-allocation-usdt.sh b/utils/verify-native-allocation-usdt.sh new file mode 100755 index 0000000000..6ec30d5503 --- /dev/null +++ b/utils/verify-native-allocation-usdt.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# Copyright 2026, Datadog, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +if [ "$#" -ne 1 ]; then + die "usage: $0 " +fi + +jar=$1 +[ -f "$jar" ] || die "JAR not found: $jar" + +for tool in unzip readelf objdump; do + command -v "$tool" >/dev/null 2>&1 || die "$tool not found" +done + +entries=$(unzip -Z1 "$jar" | grep -E '^META-INF/native-libs/linux-(x64|aarch64)/libjavaProfiler\.so$' || true) +entry_count=$(printf '%s\n' "$entries" | grep -c . || true) +[ "$entry_count" -eq 1 ] || die "expected exactly one packaged Linux profiler library, found $entry_count" +entry=$entries + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT +so="$tmp_dir/libjavaProfiler.so" +unzip -p "$jar" "$entry" >"$so" +[ -s "$so" ] || die "packaged profiler library is empty: $entry" + +machine=$(readelf -hW "$so" | sed -n 's/^[[:space:]]*Machine:[[:space:]]*//p') +case "$machine" in + *X86-64*) + expected_args='8@%rdi 8@%rsi 8@%rdx' + expected_nop='90[[:space:]]+nop' + instruction_bytes=1 + ;; + *AArch64*) + expected_args='8@x0 8@x1 8@x2' + expected_nop='d503201f[[:space:]]+nop' + instruction_bytes=4 + ;; + *) + die "unsupported ELF machine: $machine" + ;; +esac + +notes=$(readelf -nW "$so") +provider_count=$(printf '%s\n' "$notes" | grep -c 'Provider: ddheap' || true) +alloc_count=$(printf '%s\n' "$notes" | grep -c 'Name: alloc' || true) +[ "$provider_count" -eq 1 ] || die "expected one ddheap provider note, found $provider_count" +[ "$alloc_count" -eq 1 ] || die "expected one alloc probe note, found $alloc_count" +if printf '%s\n' "$notes" | grep -q 'Name: free'; then + die "unexpected ddheap:free probe in allocation-only artifact" +fi +printf '%s\n' "$notes" | grep -Fq "Arguments: $expected_args" || + die "unexpected alloc arguments; expected: $expected_args" + +location=$(printf '%s\n' "$notes" | sed -n 's/.*Location: \(0x[[:xdigit:]]*\).*/\1/p') +semaphore=$(printf '%s\n' "$notes" | sed -n 's/.*Semaphore: \(0x[[:xdigit:]]*\).*/\1/p') +[ -n "$location" ] || die "probe location is missing" +[ -n "$semaphore" ] || die "probe semaphore is missing" +(( semaphore != 0 )) || die "probe semaphore address is zero" +(( semaphore % 2 == 0 )) || die "probe semaphore is not 2-byte aligned: $semaphore" + +stop=$((location + instruction_bytes)) +instruction=$(objdump -d --start-address="$location" --stop-address="$stop" "$so") +printf '%s\n' "$instruction" | grep -Eq "$expected_nop" || + die "probe location $location is not the expected architecture NOP" + +sections=$(readelf -SW "$so") +printf '%s\n' "$sections" | grep -Fq '.note.stapsdt' || die ".note.stapsdt is missing" +if printf '%s\n' "$sections" | grep -Eq '\.debug_(info|line|abbrev|str)([[:space:]]|$)'; then + die "packaged release library still contains compiler debug sections" +fi + +echo "Verified $entry ($machine): ddheap:alloc at $location, semaphore $semaphore, args $expected_args" From 1c4df1c8c070594d88b9ad42b02c15a8106e6fcb Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 20 Oct 2026 17:04:47 +0200 Subject: [PATCH 3/5] Enable JFR and USDT native memory output by default --- ddprof-lib/src/main/cpp/arguments.h | 8 +++++- ddprof-lib/src/main/cpp/mallocTracer.cpp | 5 +++- ddprof-lib/src/main/cpp/profiler.cpp | 7 ++++++ ddprof-lib/src/test/cpp/arguments_ut.cpp | 25 +++++++++++++++---- .../src/test/cpp/mallocTracer_usdt_ut.cpp | 2 +- doc/architecture/NativeMemoryProfiling.md | 10 +++++--- utils/verify-native-allocation-usdt.sh | 11 ++++---- 7 files changed, 51 insertions(+), 17 deletions(-) diff --git a/ddprof-lib/src/main/cpp/arguments.h b/ddprof-lib/src/main/cpp/arguments.h index 04037fe246..e2752497bd 100644 --- a/ddprof-lib/src/main/cpp/arguments.h +++ b/ddprof-lib/src/main/cpp/arguments.h @@ -118,6 +118,12 @@ enum NativeMemoryOutput { NATIVE_MEMORY_BOTH }; +#if defined(__linux__) +const NativeMemoryOutput DEFAULT_NATIVE_MEMORY_OUTPUT = NATIVE_MEMORY_BOTH; +#else +const NativeMemoryOutput DEFAULT_NATIVE_MEMORY_OUTPUT = NATIVE_MEMORY_JFR; +#endif + enum Style { STYLE_SIMPLE = 1, STYLE_DOTTED = 2, @@ -313,7 +319,7 @@ class Arguments { _reference_chains_first_pass_budget(DEFAULT_REFERENCE_CHAINS_FIRST_PASS_BUDGET), _reference_chains_tuned_mask(0), _nativemem(-1), - _nativemem_output(NATIVE_MEMORY_JFR), + _nativemem_output(DEFAULT_NATIVE_MEMORY_OUTPUT), _nativemem_output_set(false), _jstackdepth(DEFAULT_JSTACKDEPTH), _safe_mode(0), diff --git a/ddprof-lib/src/main/cpp/mallocTracer.cpp b/ddprof-lib/src/main/cpp/mallocTracer.cpp index 0e89f16d46..ca0d5169f6 100644 --- a/ddprof-lib/src/main/cpp/mallocTracer.cpp +++ b/ddprof-lib/src/main/cpp/mallocTracer.cpp @@ -10,6 +10,7 @@ #include #include #include "codeCache.h" +#include "common.h" #include "counters.h" #include "guards.h" #include "libraries.h" @@ -482,7 +483,9 @@ Error MallocTracer::check(Arguments& args) { } #if !defined(__linux__) if (args._nativemem_output != NATIVE_MEMORY_JFR) { - return Error("nativememoutput=usdt and both are supported only on Linux"); + const char* output = args._nativemem_output == NATIVE_MEMORY_USDT ? "usdt" : "both"; + LOG_WARN("nativememoutput=%s is supported only on Linux; falling back to JFR", output); + args._nativemem_output = NATIVE_MEMORY_JFR; } #endif return Error::OK; diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 1c5ab4e168..138944b0e7 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1486,6 +1486,13 @@ Error Profiler::start(Arguments &args, bool reset) { return error; } + if (args._nativemem >= 0 || args._nativemem_output_set) { + error = malloc_tracer.check(args); + if (error) { + return error; + } + } + _omit_stacktraces = args._lightweight; _remote_symbolication = args._remote_symbolication; _libs->setRemoteSymbolication(_remote_symbolication); diff --git a/ddprof-lib/src/test/cpp/arguments_ut.cpp b/ddprof-lib/src/test/cpp/arguments_ut.cpp index 50c90f6001..8acc762e95 100644 --- a/ddprof-lib/src/test/cpp/arguments_ut.cpp +++ b/ddprof-lib/src/test/cpp/arguments_ut.cpp @@ -29,14 +29,17 @@ class ArgumentsTest : public ::testing::Test { void TearDown() override {} }; -TEST_F(ArgumentsTest, NativeMemoryOutputDefaultsToJfrAndIsNotExplicit) { +TEST_F(ArgumentsTest, NativeMemoryOutputDefaultsForThePlatformAndIsNotExplicit) { Arguments args; - EXPECT_EQ(args._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_EQ(args._nativemem_output, DEFAULT_NATIVE_MEMORY_OUTPUT); EXPECT_FALSE(args._nativemem_output_set); ASSERT_FALSE(args.parse("nativemem=512")); - EXPECT_EQ(args._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_EQ(args._nativemem_output, DEFAULT_NATIVE_MEMORY_OUTPUT); EXPECT_FALSE(args._nativemem_output_set); + + MallocTracer tracer; + EXPECT_FALSE(tracer.check(args)); } TEST_F(ArgumentsTest, NativeMemoryOutputParsesAllValuesCaseInsensitively) { @@ -78,16 +81,28 @@ TEST_F(ArgumentsTest, NativeMemoryOutputRequiresNativeMemoryRegardlessOfOrder) { ASSERT_FALSE(output_first.parse("nativememoutput=usdt,nativemem=512")); #if defined(__linux__) EXPECT_FALSE(tracer.check(output_first)); + EXPECT_EQ(output_first._nativemem_output, NATIVE_MEMORY_USDT); #else - EXPECT_TRUE(tracer.check(output_first)); + testing::internal::CaptureStderr(); + EXPECT_FALSE(tracer.check(output_first)); + std::string usdt_warning = testing::internal::GetCapturedStderr(); + EXPECT_EQ(output_first._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_NE(std::string::npos, usdt_warning.find( + "nativememoutput=usdt is supported only on Linux; falling back to JFR")); #endif Arguments memory_first; ASSERT_FALSE(memory_first.parse("nativemem=512,nativememoutput=both")); #if defined(__linux__) EXPECT_FALSE(tracer.check(memory_first)); + EXPECT_EQ(memory_first._nativemem_output, NATIVE_MEMORY_BOTH); #else - EXPECT_TRUE(tracer.check(memory_first)); + testing::internal::CaptureStderr(); + EXPECT_FALSE(tracer.check(memory_first)); + std::string both_warning = testing::internal::GetCapturedStderr(); + EXPECT_EQ(memory_first._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_NE(std::string::npos, both_warning.find( + "nativememoutput=both is supported only on Linux; falling back to JFR")); #endif } diff --git a/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp b/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp index 0f14af772b..c7838abb7f 100644 --- a/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp +++ b/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp @@ -162,7 +162,7 @@ TEST_F(MallocTracerUsdtTest, EveryNonzeroSemaphoreValueIsActive) { #if defined(__linux__) TEST_F(MallocTracerUsdtTest, SemaphoreUsesTheKernelReferenceCountSemantics) { - for (uint16_t value : {uint16_t(0), uint16_t(1), uint16_t(2), UINT16_MAX}) { + for (uint16_t value : {uint16_t(0), uint16_t(1), uint16_t(2), uint16_t(UINT16_MAX)}) { setNativeAllocationUsdtSemaphoreForTest(value); EXPECT_EQ(value != 0, isNativeAllocationUsdtEnabled()); } diff --git a/doc/architecture/NativeMemoryProfiling.md b/doc/architecture/NativeMemoryProfiling.md index 2b0800f175..d6f90c3a67 100644 --- a/doc/architecture/NativeMemoryProfiling.md +++ b/doc/architecture/NativeMemoryProfiling.md @@ -12,16 +12,18 @@ requiring a custom allocator. Free events are not intercepted or recorded. By default, sampled allocation events carry a full Java + native stack trace and are emitted as `datadog.NativeMemoryAllocation` JFR events. Linux x86-64 and -AArch64 builds can instead expose sampled allocations through the semaphored +AArch64 builds also expose sampled allocations through the semaphored `ddheap:alloc(address, size, weighted_bytes)` USDT probe for an external eBPF profiler. The feature is activated by passing `nativemem=` to the profiler, where `` is the byte-sampling interval (e.g. `nativemem=524288` samples roughly one event per 512 KiB allocated). Passing `nativemem=0` records every allocation. -`nativememoutput=jfr|usdt|both` selects the destination; `jfr` is the default. -The output option does not activate profiling by itself and `usdt`/`both` are -rejected outside Linux. +`nativememoutput=jfr|usdt|both` selects the destination; `both` is the default. +Pass `jfr` to disable USDT output or `usdt` to disable JFR output. The output +option does not activate profiling by itself. On non-Linux platforms the implicit +default emits JFR only. Explicit `usdt`/`both` values emit a warning and fall back +to JFR. USDT output is allocation-only. It deliberately does not add a free probe or retain live-heap state. `weighted_bytes` is the allocation size divided by its diff --git a/utils/verify-native-allocation-usdt.sh b/utils/verify-native-allocation-usdt.sh index 6ec30d5503..23b0d0875e 100755 --- a/utils/verify-native-allocation-usdt.sh +++ b/utils/verify-native-allocation-usdt.sh @@ -51,17 +51,18 @@ esac notes=$(readelf -nW "$so") provider_count=$(printf '%s\n' "$notes" | grep -c 'Provider: ddheap' || true) -alloc_count=$(printf '%s\n' "$notes" | grep -c 'Name: alloc' || true) [ "$provider_count" -eq 1 ] || die "expected one ddheap provider note, found $provider_count" +ddheap_note=$(printf '%s\n' "$notes" | grep -A3 'Provider: ddheap') +alloc_count=$(printf '%s\n' "$ddheap_note" | grep -c 'Name: alloc' || true) [ "$alloc_count" -eq 1 ] || die "expected one alloc probe note, found $alloc_count" -if printf '%s\n' "$notes" | grep -q 'Name: free'; then +if printf '%s\n' "$ddheap_note" | grep -q 'Name: free'; then die "unexpected ddheap:free probe in allocation-only artifact" fi -printf '%s\n' "$notes" | grep -Fq "Arguments: $expected_args" || +printf '%s\n' "$ddheap_note" | grep -Fq "Arguments: $expected_args" || die "unexpected alloc arguments; expected: $expected_args" -location=$(printf '%s\n' "$notes" | sed -n 's/.*Location: \(0x[[:xdigit:]]*\).*/\1/p') -semaphore=$(printf '%s\n' "$notes" | sed -n 's/.*Semaphore: \(0x[[:xdigit:]]*\).*/\1/p') +location=$(printf '%s\n' "$ddheap_note" | sed -n 's/.*Location: \(0x[[:xdigit:]]*\).*/\1/p') +semaphore=$(printf '%s\n' "$ddheap_note" | sed -n 's/.*Semaphore: \(0x[[:xdigit:]]*\).*/\1/p') [ -n "$location" ] || die "probe location is missing" [ -n "$semaphore" ] || die "probe semaphore is missing" (( semaphore != 0 )) || die "probe semaphore address is zero" From 400fafabb79579666801fb1c90336832d9f62bfb Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 20 Oct 2026 17:19:37 +0200 Subject: [PATCH 4/5] Use no-op fallback for unsupported USDT output --- ddprof-lib/src/main/cpp/arguments.h | 3 ++- ddprof-lib/src/main/cpp/mallocTracer.cpp | 19 +++++++++++----- ddprof-lib/src/test/cpp/arguments_ut.cpp | 7 +++--- .../src/test/cpp/mallocTracer_usdt_ut.cpp | 22 +++++++++++++++++++ doc/architecture/NativeMemoryProfiling.md | 4 ++-- 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/ddprof-lib/src/main/cpp/arguments.h b/ddprof-lib/src/main/cpp/arguments.h index e2752497bd..aafaa1cf24 100644 --- a/ddprof-lib/src/main/cpp/arguments.h +++ b/ddprof-lib/src/main/cpp/arguments.h @@ -115,7 +115,8 @@ enum Ring { enum NativeMemoryOutput { NATIVE_MEMORY_JFR, NATIVE_MEMORY_USDT, - NATIVE_MEMORY_BOTH + NATIVE_MEMORY_BOTH, + NATIVE_MEMORY_NOOP }; #if defined(__linux__) diff --git a/ddprof-lib/src/main/cpp/mallocTracer.cpp b/ddprof-lib/src/main/cpp/mallocTracer.cpp index ca0d5169f6..ed4a710e10 100644 --- a/ddprof-lib/src/main/cpp/mallocTracer.cpp +++ b/ddprof-lib/src/main/cpp/mallocTracer.cpp @@ -382,7 +382,7 @@ void MallocTracer::recordMalloc(void* address, size_t size, int tid) { SAMPLER_PERF_PROBE(SP_NATIVEMEM); u64 current_interval = __atomic_load_n(&_interval, __ATOMIC_ACQUIRE); - if (_output != NATIVE_MEMORY_USDT) { + if (_output == NATIVE_MEMORY_JFR || _output == NATIVE_MEMORY_BOTH) { MallocEvent event; event._start_time = TSC::ticks(); event._address = (uintptr_t)address; @@ -407,7 +407,8 @@ void MallocTracer::recordMalloc(void* address, size_t size, int tid) { } bool usdt_active = false; - if (_output != NATIVE_MEMORY_JFR && size <= (size_t)INT64_MAX) { + if ((_output == NATIVE_MEMORY_USDT || _output == NATIVE_MEMORY_BOTH) && + size <= (size_t)INT64_MAX) { #ifdef UNIT_TEST usdt_active = _test_usdt_active == NULL ? isNativeAllocationUsdtEnabled() : _test_usdt_active(); @@ -452,6 +453,11 @@ void MallocTracer::recordMalloc(void* address, size_t size, int tid) { Error MallocTracer::start(Arguments& args) { _output = args._nativemem_output; + if (_output == NATIVE_MEMORY_NOOP) { + __atomic_store_n(&_running, false, __ATOMIC_RELEASE); + return Error::OK; + } + _configured_interval = args._nativemem > 0 ? args._nativemem : 0; __atomic_store_n(&_interval, _configured_interval, __ATOMIC_RELEASE); __atomic_store_n(&_bytes_until_sample, @@ -482,9 +488,12 @@ Error MallocTracer::check(Arguments& args) { return Error("nativememoutput requires nativemem"); } #if !defined(__linux__) - if (args._nativemem_output != NATIVE_MEMORY_JFR) { - const char* output = args._nativemem_output == NATIVE_MEMORY_USDT ? "usdt" : "both"; - LOG_WARN("nativememoutput=%s is supported only on Linux; falling back to JFR", output); + if (args._nativemem_output == NATIVE_MEMORY_USDT) { + LOG_WARN("nativememoutput=usdt is supported only on Linux; " + "native memory profiling will be disabled"); + args._nativemem_output = NATIVE_MEMORY_NOOP; + } else if (args._nativemem_output == NATIVE_MEMORY_BOTH) { + LOG_WARN("USDT output is supported only on Linux; continuing with JFR only"); args._nativemem_output = NATIVE_MEMORY_JFR; } #endif diff --git a/ddprof-lib/src/test/cpp/arguments_ut.cpp b/ddprof-lib/src/test/cpp/arguments_ut.cpp index 8acc762e95..a80be3c027 100644 --- a/ddprof-lib/src/test/cpp/arguments_ut.cpp +++ b/ddprof-lib/src/test/cpp/arguments_ut.cpp @@ -86,9 +86,10 @@ TEST_F(ArgumentsTest, NativeMemoryOutputRequiresNativeMemoryRegardlessOfOrder) { testing::internal::CaptureStderr(); EXPECT_FALSE(tracer.check(output_first)); std::string usdt_warning = testing::internal::GetCapturedStderr(); - EXPECT_EQ(output_first._nativemem_output, NATIVE_MEMORY_JFR); + EXPECT_EQ(output_first._nativemem_output, NATIVE_MEMORY_NOOP); EXPECT_NE(std::string::npos, usdt_warning.find( - "nativememoutput=usdt is supported only on Linux; falling back to JFR")); + "nativememoutput=usdt is supported only on Linux; " + "native memory profiling will be disabled")); #endif Arguments memory_first; @@ -102,7 +103,7 @@ TEST_F(ArgumentsTest, NativeMemoryOutputRequiresNativeMemoryRegardlessOfOrder) { std::string both_warning = testing::internal::GetCapturedStderr(); EXPECT_EQ(memory_first._nativemem_output, NATIVE_MEMORY_JFR); EXPECT_NE(std::string::npos, both_warning.find( - "nativememoutput=both is supported only on Linux; falling back to JFR")); + "USDT output is supported only on Linux; continuing with JFR only")); #endif } diff --git a/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp b/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp index c7838abb7f..4a3cfdefd1 100644 --- a/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp +++ b/ddprof-lib/src/test/cpp/mallocTracer_usdt_ut.cpp @@ -123,6 +123,28 @@ TEST_F(MallocTracerUsdtTest, JfrModeNeverQueriesOrEmitsUsdt) { expectStages({"jfr", "accounting"}); } +TEST_F(MallocTracerUsdtTest, NoopModeProducesNoOutput) { + active_value = 1; + MallocTracer::configureForTest(NATIVE_MEMORY_NOOP, 512, true); + MallocTracer::recordMalloc(reinterpret_cast(0x1234), 512, 7); + + EXPECT_EQ(0, jfr_count); + EXPECT_EQ(0, active_count); + EXPECT_EQ(0, emit_count); + expectStages({"accounting"}); +} + +TEST_F(MallocTracerUsdtTest, NoopModeDoesNotStartTracer) { + Arguments args; + args._nativemem = 512; + args._nativemem_output = NATIVE_MEMORY_NOOP; + MallocTracer tracer; + + EXPECT_FALSE(tracer.start(args)); + EXPECT_FALSE(MallocTracer::running()); + expectStages({}); +} + TEST_F(MallocTracerUsdtTest, UsdtModeSkipsJfrAndEmitsWeightedPayload) { active_value = 2; void* address = reinterpret_cast(0x1234); diff --git a/doc/architecture/NativeMemoryProfiling.md b/doc/architecture/NativeMemoryProfiling.md index d6f90c3a67..2dfebcffb2 100644 --- a/doc/architecture/NativeMemoryProfiling.md +++ b/doc/architecture/NativeMemoryProfiling.md @@ -22,8 +22,8 @@ one event per 512 KiB allocated). Passing `nativemem=0` records every allocation `nativememoutput=jfr|usdt|both` selects the destination; `both` is the default. Pass `jfr` to disable USDT output or `usdt` to disable JFR output. The output option does not activate profiling by itself. On non-Linux platforms the implicit -default emits JFR only. Explicit `usdt`/`both` values emit a warning and fall back -to JFR. +default emits JFR only. Explicit `usdt` emits a warning and disables native-memory +profiling, while explicit `both` emits a warning and continues with JFR only. USDT output is allocation-only. It deliberately does not add a free probe or retain live-heap state. `weighted_bytes` is the allocation size divided by its From 08ebfd8c707b2053c504cb8483d05e79ba09d362 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 20 Oct 2026 17:42:44 +0200 Subject: [PATCH 5/5] Update README.md --- ddprof-lib/src/main/cpp/vendor/libbpf/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ddprof-lib/src/main/cpp/vendor/libbpf/README.md b/ddprof-lib/src/main/cpp/vendor/libbpf/README.md index a231024104..5b9fed1cef 100644 --- a/ddprof-lib/src/main/cpp/vendor/libbpf/README.md +++ b/ddprof-lib/src/main/cpp/vendor/libbpf/README.md @@ -3,9 +3,7 @@ # Vendored libbpf USDT header `usdt.h` is copied without modification from the version vendored by -libdatadog at commit `c3b0ee0a3f8a45e5a0cf153645ceffee6bbe7ec8`: - -https://raw.githubusercontent.com/DataDog/libdatadog/c3b0ee0a3f8a45e5a0cf153645ceffee6bbe7ec8/libdd-profiling-heap-sampler/vendor/usdt.h +[libdatadog at commit `c3b0ee0a3f8a45e5a0cf153645ceffee6bbe7ec8`](https://raw.githubusercontent.com/DataDog/libdatadog/c3b0ee0a3f8a45e5a0cf153645ceffee6bbe7ec8/libdd-profiling-heap-sampler/vendor/usdt.h) - Length: 25,031 bytes - SHA-256: `5621bc8ae809d6e52f7833998194351f6b71c6fe83496b19e0d359757d18f8ee`