Fix native OOM from unbounded CallTraceHashTable expansion

ddprof-lib/src/main/cpp/callTraceHashTable.cpp

@@ -30,14 +30,14 @@ class LongHashTable {
   volatile u32 _size;
   u32 _padding2[15];
 
+public:
   static size_t getSize(u32 capacity) {
     size_t size = sizeof(LongHashTable) +
                   (sizeof(u64) + sizeof(CallTraceSample)) * capacity;
     return (size + OS::page_mask) & ~OS::page_mask;
   }
 
-public:
-  LongHashTable(LongHashTable *prev = nullptr, u32 capacity = 0, bool should_clean = true) 
+  LongHashTable(LongHashTable *prev = nullptr, u32 capacity = 0, bool should_clean = true)
     : _prev(prev), _padding0(nullptr), _capacity(capacity), _size(0) {
     memset(_padding1, 0, sizeof(_padding1));
     memset(_padding2, 0, sizeof(_padding2));
@@ -280,6 +280,19 @@ void CallTraceHashTable::expandTableIfNeeded(LongHashTable* table, u32 size) {
   // EXPANSION LOGIC: Check if load ratio reached after incrementing size
   if (size >= (u32) (capacity * LOAD_RATIO)  &&
       table == __atomic_load_n(&_table, __ATOMIC_RELAXED)) { // quick check, if other thread already expanded the table
+    if (LongHashTable::getSize(capacity * 2) > _allocator.maxAllocatableSize()) {
+      // The doubled table would not fit in a single LinearAllocator chunk.
+      // alloc() bump-allocates within one chunk and cannot satisfy a request
+      // larger than a chunk - it would loop allocating fresh chunks until mmap
+      // fails (native OOM / vm.max_map_count exhaustion). Stop expanding here;
+      // put() keeps working on the current table at a higher load factor. This
+      // degraded state is bounded, not permanent: the next processTraces()
+      // rotation resets the table to its initial capacity, so normal JFR-flush
+      // cadence recovers it. Deriving the bound from the chunk size keeps it
+      // correct if the chunk size ever changes.
+      Counters::increment(CALLTRACE_STORAGE_EXPANSION_SKIPPED);
+      return;
+    }
     // Allocate new table with double capacity using LinearAllocator
     LongHashTable* new_table = LongHashTable::allocate(table, capacity * 2, &_allocator);
     if (new_table != nullptr) {

ddprof-lib/src/main/cpp/counters.h

@@ -72,6 +72,7 @@
   X(UNWINDING_TIME_ASYNC, "unwinding_ticks_async")                             \
   X(UNWINDING_TIME_JVMTI, "unwinding_ticks_jvmti")                             \
   X(CALLTRACE_STORAGE_DROPPED, "calltrace_storage_dropped_traces")             \
+  X(CALLTRACE_STORAGE_EXPANSION_SKIPPED, "calltrace_storage_expansion_skipped")\
   X(LINE_NUMBER_TABLES, "line_number_tables")                                  \
   X(REMOTE_SYMBOLICATION_FRAMES, "remote_symbolication_frames")                \
   X(REMOTE_SYMBOLICATION_LIBS_WITH_BUILD_ID, "remote_symbolication_libs_with_build_id") \

ddprof-lib/src/main/cpp/linearAllocator.h

@@ -73,6 +73,12 @@ class LinearAllocator {
   static void freeChunks(ChunkList& chunks);
 
   void *alloc(size_t size);
+
+  // Largest single allocation this allocator can ever satisfy: one chunk minus
+  // its header. alloc() bump-allocates within a single chunk, so a request
+  // larger than this can never be placed - callers must not request more (see
+  // CallTraceHashTable::expandTableIfNeeded()).
+  size_t maxAllocatableSize() const { return _chunk_size - sizeof(Chunk); }
 };
 
 #endif // _LINEARALLOCATOR_H

ddprof-lib/src/main/cpp/livenessTracker.cpp

@@ -300,6 +300,27 @@ static void free_uniform_real_distribution(void* p) {
   delete urd;
 }
 
+// File-scope (not track()-local) so releaseThreadLocalState() below can reach
+// them from Profiler::onThreadEnd(). Relying solely on these ThreadLocal's own
+// pthread-key destructors is not sufficient: pthread key destructors only fire
+// when the underlying OS thread actually exits, not when a JNI-attached thread
+// detaches via DetachCurrentThread. A reused pooled OS thread that repeatedly
+// attaches/detaches would otherwise leak one mt19937 and one
+// uniform_real_distribution allocation per attach cycle, since get() lazily
+// re-creates the value on the next track() call but nothing ever frees the
+// previous one until OS thread exit (which may never happen). Hooking explicit
+// cleanup into onThreadEnd matches how every other per-thread profiler state
+// (CPU/wall engine registration, ProfiledThread) is already torn down.
+static ThreadLocal<std::mt19937*, create_mt19937, free_mt19937> gen;
+static ThreadLocal<std::uniform_real_distribution<>*, create_uniform_real_distribution, free_uniform_real_distribution> dis;
+static ThreadLocal<double> skipped;
+
+void LivenessTracker::releaseThreadLocalState() {
+  gen.clear();
+  dis.clear();
+  skipped.clear();
+}
+
 void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid,
                             jobject object, u64 call_trace_id) {
   if (!_enabled) {
@@ -311,10 +332,6 @@ void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid,
     return;
   }
 
-  static ThreadLocal<std::mt19937*, create_mt19937, free_mt19937> gen;
-  static ThreadLocal<std::uniform_real_distribution<>*, create_uniform_real_distribution, free_uniform_real_distribution> dis;
-  static ThreadLocal<double> skipped;
-
   if (_subsample_ratio < 1.0) {
     std::mt19937* genp = gen.get();
     std::uniform_real_distribution<>* disp = dis.get();

ddprof-lib/src/main/cpp/livenessTracker.h

@@ -94,6 +94,13 @@ class alignas(alignof(SpinLock)) LivenessTracker {
   void track(JNIEnv *env, AllocEvent &event, jint tid, jobject object, u64 call_trace_id);
   void flush(std::set<int> &tracked_thread_ids);
 
+  // Frees this thread's subsampling RNG state (track()'s gen/dis/skipped
+  // ThreadLocals, livenessTracker.cpp). Must be called from a thread that is
+  // about to detach/terminate - see those ThreadLocal's own comment for why
+  // their pthread-key destructors alone cannot be relied on for JNI-attached
+  // threads. Safe to call even if this thread never called track().
+  static void releaseThreadLocalState();
+
   static void JNICALL GarbageCollectionFinish(jvmtiEnv *jvmti_env);
 
 private:

ddprof-lib/src/main/cpp/profiler.cpp

@@ -115,6 +115,7 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) {
       SignalBlocker blocker;
       _cpu_engine->unregisterThread(tid);
       _wall_engine->unregisterThread(tid);
+      LivenessTracker::instance()->releaseThreadLocalState();
       ProfiledThread::release();
     }
     return;
@@ -130,6 +131,7 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) {
   updateThreadName(jvmti, jni, thread, false);
   _cpu_engine->unregisterThread(tid);
   _wall_engine->unregisterThread(tid);
+  LivenessTracker::instance()->releaseThreadLocalState();
 }
 
 int Profiler::registerThread(int tid) {