diff --git a/tcmalloc/allocation_sampling.cc b/tcmalloc/allocation_sampling.cc index 65115749d..5af8d2358 100644 --- a/tcmalloc/allocation_sampling.cc +++ b/tcmalloc/allocation_sampling.cc @@ -58,19 +58,5 @@ std::unique_ptr DumpHeapProfile(Static& state) { return profile; } -ABSL_ATTRIBUTE_NOINLINE void FreeProxyObject(Static& state, void* ptr, - size_t size_class) { - if (ABSL_PREDICT_TRUE(UsePerCpuCache(state))) { - state.cpu_cache().Deallocate(ptr, size_class); - } else if (ThreadCache* cache = ThreadCache::GetCacheIfPresent(); - ABSL_PREDICT_TRUE(cache)) { - cache->Deallocate(ptr, size_class); - } else { - // This thread doesn't have thread-cache yet or already. Delete directly - // into transfer cache. - state.transfer_cache().InsertRange(size_class, absl::Span(&ptr, 1)); - } -} - } // namespace tcmalloc::tcmalloc_internal GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 9753aa7b7..7db362dcc 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -67,16 +67,9 @@ inline Sampler& GetThreadSampler() { #endif } -void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); - // Performs sampling for already occurred allocation of object. // -// For very small object sizes, object is used as 'proxy' and full -// page with sampled marked is allocated instead. -// -// For medium-sized objects that have single instance per span, -// they're simply freed and fresh page span is allocated to represent -// sampling. +// For small object sizes, we allocate a new object in a sampled span. // // For large objects (i.e. allocated with do_malloc_pages) they are // also fully reused and their span is marked as sampled. @@ -95,12 +88,10 @@ void FreeProxyObject(Static& state, void* absl_nonnull ptr, size_t size_class); template ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, void* absl_nullable obj, Span* absl_nullable span) { - TC_CHECK((size_class != 0 && obj != nullptr && span == nullptr) || - (size_class == 0 && obj == nullptr && span != nullptr)); + size_t size_class, Span* absl_nullable span) { + TC_CHECK_EQ(size_class != 0, span == nullptr); StackTrace stack_trace; - stack_trace.proxy = nullptr; stack_trace.requested_size = requested_size; // Grab the stack trace outside the heap lock. stack_trace.depth = absl::GetStackTrace(stack_trace.stack, kMaxStackDepth, 0); @@ -131,8 +122,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( : MemoryTag::kSampled; size_t capacity = 0; if (size_class != 0) { - TC_ASSERT_EQ(size_class, - state.pagemap().sizeclass(PageIdContainingTagged(obj))); state.per_size_class_counts()[size_class].Add(allocation_estimate); stack_trace.allocated_size = state.sizemap().class_to_size(size_class); @@ -161,24 +150,10 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( } capacity = requested_size; } else if ((span = state.page_allocator().New( - num_pages, {1, AccessDensityPrediction::kSparse}, tag)) == - nullptr) { + num_pages, {1, AccessDensityPrediction::kSparse}, tag))) { capacity = stack_trace.allocated_size; - return {obj, capacity}; } else { - capacity = stack_trace.allocated_size; - } - - size_t span_size = - Length(state.sizemap().class_to_pages(size_class)).in_bytes(); - size_t objects_per_span = stack_trace.allocated_size != 0 - ? span_size / stack_trace.allocated_size - : -1ul; - - if (objects_per_span != 1) { - TC_ASSERT_GT(objects_per_span, 1); - stack_trace.proxy = obj; - obj = nullptr; + return {}; } } else { // Set stack_trace.allocated_size to the exact size for a page allocation. @@ -244,12 +219,6 @@ ABSL_ATTRIBUTE_NOINLINE sized_ptr_t SampleifyAllocation( state.peak_heap_tracker().MaybeSaveSample(); - if (obj != nullptr) { - // We are not maintaining precise statistics on malloc hit/miss rates at our - // cache tiers. We can deallocate into our ordinary cache. - TC_ASSERT_NE(size_class, 0); - FreeProxyObject(state, obj, size_class); - } TC_ASSERT_EQ(state.pagemap().sizeclass(span->first_page()), 0); return {(alloc_with_status.alloc != nullptr) ? alloc_with_status.alloc : span->start_address(), @@ -264,16 +233,15 @@ template static sized_ptr_t SampleLargeAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, Span* span) { - return SampleifyAllocation(state, policy, requested_size, weight, 0, nullptr, - span); + return SampleifyAllocation(state, policy, requested_size, weight, 0, span); } template static sized_ptr_t SampleSmallAllocation(Static& state, Policy policy, size_t requested_size, size_t weight, - size_t size_class, sized_ptr_t res) { + size_t size_class) { return SampleifyAllocation(state, policy, requested_size, weight, size_class, - res.p, nullptr); + nullptr); } // Rewrite type so that the allocation type falls into one of the categories we @@ -323,7 +291,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, TC_ASSERT_EQ(state.pagemap().sizeclass(PageIdContainingTagged(ptr)), 0); - void* const proxy = sampled_allocation->sampled_stack.proxy; const size_t weight = sampled_allocation->sampled_stack.weight; const size_t requested_size = sampled_allocation->sampled_stack.requested_size; @@ -366,18 +333,14 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, sampled_allocation->sampled_stack.depth)); } - const size_t allocated_alignment = static_cast( - sampled_allocation->sampled_stack.requested_alignment.value_or( - std::align_val_t{1})); - const std::optional deallocated_alignment = - policy.has_explicit_alignment() - ? std::make_optional(policy.align()) - : std::nullopt; - if ((size.has_value() || policy.allocation_type() == AllocationType::New)) { const bool type_mismatch = policy.allocation_type() != sampled_allocation->sampled_stack.allocation_type; + const std::optional deallocated_alignment = + policy.has_explicit_alignment() + ? std::make_optional(policy.align()) + : std::nullopt; const bool alignment_mismatch = deallocated_alignment != sampled_allocation->sampled_stack.requested_alignment; @@ -428,23 +391,6 @@ void MaybeUnsampleAllocation(Static& state, Policy policy, MallocHook::InvokeSampledDeleteHook(sampled_alloc); state.deallocation_samples.ReportFree(sampled_alloc_handle); - - if (proxy) { - const auto proxy_policy = policy.InSamePartitionAs(proxy); - size_t size_class; - if (AccessFromPointer(proxy) == AllocationAccess::kCold) { - size_class = state.sizemap().SizeClass( - proxy_policy.AccessAsCold().AlignAs(allocated_alignment), - allocated_size); - } else { - size_class = state.sizemap().SizeClass( - proxy_policy.AccessAsHot().AlignAs(allocated_alignment), - allocated_size); - } - TC_ASSERT_EQ(size_class, - state.pagemap().sizeclass(PageIdContainingTagged(proxy))); - FreeProxyObject(state, proxy, size_class); - } } } // namespace tcmalloc::tcmalloc_internal diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 4e7165c73..704861807 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -248,17 +248,6 @@ enum class AllocationAccess { kCold, }; -inline AllocationAccess AccessFromPointer(void* ptr) { - if (!kHasExpandedClasses) { - TC_ASSERT_NE(GetMemoryTag(ptr), MemoryTag::kCold); - return AllocationAccess::kHot; - } - - return ABSL_PREDICT_FALSE(GetMemoryTag(ptr) == MemoryTag::kCold) - ? AllocationAccess::kCold - : AllocationAccess::kHot; -} - inline MemoryTag MultiNormalTag(size_t partition) { switch (partition) { case 0: diff --git a/tcmalloc/experimental_pow2_size_class.cc b/tcmalloc/experimental_pow2_size_class.cc index be170038b..820be1e33 100644 --- a/tcmalloc/experimental_pow2_size_class.cc +++ b/tcmalloc/experimental_pow2_size_class.cc @@ -132,7 +132,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 @@ -251,7 +251,7 @@ static constexpr SizeClassInfo List[] = { { 16384, 262144, 4}, // 12 16 0.02% 12.53% 100.00% { 32768, 262144, 2}, // 13 8 0.02% 12.53% 100.00% { 65536, 262144, 2}, // 14 4 0.02% 12.53% 100.00% - {131072, 262144, 2}, // 15 2 0.02% 12.54% 100.00% + {131072, 262144, 2}, // 15 2 0.02% 12.53% 100.00% {262144, 262144, 2}, // 16 1 0.02% 0.03% 100.00% }; #elif TCMALLOC_PAGE_SHIFT == 12 diff --git a/tcmalloc/internal/logging.h b/tcmalloc/internal/logging.h index f2b675bc2..c0cd16d04 100644 --- a/tcmalloc/internal/logging.h +++ b/tcmalloc/internal/logging.h @@ -64,15 +64,6 @@ struct StackTrace { // memory block. AllocHandle sampled_alloc_handle; - // For small sampled objects, we allocate a full span to hold the - // sampled object. However to avoid disturbing fragmentation - // profiles, in such cases we also allocate a small proxy object - // using the normal mechanism. - // - // proxy field is defined only for heap sample stack traces. - // For heap samples, proxy==NULL iff size > kMaxSize. - void* proxy; - uintptr_t requested_size; std::optional requested_alignment; uintptr_t allocated_size; // size after sizeclass/page rounding diff --git a/tcmalloc/legacy_size_classes.cc b/tcmalloc/legacy_size_classes.cc index 26d70179e..38f70e817 100644 --- a/tcmalloc/legacy_size_classes.cc +++ b/tcmalloc/legacy_size_classes.cc @@ -98,9 +98,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 38 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 39 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 40 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 41 9 2.33% 0.43% 16.67% + { 896, 8192, 32}, // 41 9 2.33% 0.42% 16.67% { 1024, 8192, 32}, // 42 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 43 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 43 14 1.95% 0.42% 12.50% { 1280, 16384, 32}, // 44 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 45 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 46 10 6.61% 0.43% 9.09% @@ -228,7 +228,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -296,17 +296,17 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 46 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 47 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 48 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 49 81 1.15% 12.54% 13.64% + { 3200, 262144, 20}, // 49 81 1.15% 12.53% 13.64% { 3584, 262144, 18}, // 50 73 0.22% 12.53% 12.00% { 4096, 262144, 16}, // 51 64 0.02% 12.53% 14.29% { 4224, 262144, 15}, // 52 62 0.12% 12.53% 3.12% - { 4736, 262144, 13}, // 53 55 0.66% 12.54% 12.12% + { 4736, 262144, 13}, // 53 55 0.66% 12.53% 12.12% { 5120, 262144, 12}, // 54 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 55 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 56 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 57 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 58 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 59 34 0.41% 12.54% 9.09% + { 7680, 262144, 8}, // 59 34 0.41% 12.53% 9.09% { 8192, 262144, 8}, // 60 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 61 28 0.22% 12.53% 14.06% { 11392, 262144, 5}, // 62 23 0.07% 12.53% 21.92% @@ -330,7 +330,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% @@ -439,7 +439,7 @@ static constexpr SizeClassInfo List[] = { { 368, 8192, 32}, // 24 22 1.94% 0.42% 4.55% { 384, 8192, 32}, // 25 21 2.33% 0.42% 4.35% { 400, 8192, 32}, // 26 20 3.10% 0.42% 4.17% - { 416, 8192, 32}, // 27 19 4.26% 0.43% 4.00% + { 416, 8192, 32}, // 27 19 4.26% 0.42% 4.00% { 448, 8192, 32}, // 28 18 2.33% 0.42% 7.69% { 480, 8192, 32}, // 29 17 1.16% 0.42% 7.14% { 512, 8192, 32}, // 30 16 0.78% 0.42% 6.67% @@ -447,9 +447,9 @@ static constexpr SizeClassInfo List[] = { { 640, 8192, 32}, // 32 12 6.98% 0.43% 11.11% { 704, 8192, 32}, // 33 11 6.20% 0.43% 10.00% { 768, 8192, 32}, // 34 10 6.98% 0.43% 9.09% - { 896, 8192, 32}, // 35 9 2.33% 0.43% 16.67% + { 896, 8192, 32}, // 35 9 2.33% 0.42% 16.67% { 1024, 8192, 32}, // 36 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 37 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 37 14 1.95% 0.42% 12.50% { 1280, 16384, 32}, // 38 12 6.61% 0.43% 11.11% { 1408, 16384, 32}, // 39 11 5.84% 0.43% 10.00% { 1536, 16384, 32}, // 40 10 6.61% 0.43% 9.09% @@ -583,7 +583,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 69 1 0.10% 0.03% 14.29% { 81920, 163840, 2}, // 70 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 71 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 72 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 72 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 73 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 74 1 0.04% 0.03% 25.00% {196608, 196608, 2}, // 75 1 0.03% 0.03% 20.00% @@ -647,19 +647,19 @@ static constexpr SizeClassInfo List[] = { { 2304, 262144, 28}, // 42 113 0.71% 12.53% 12.50% { 2560, 262144, 25}, // 43 102 0.41% 12.53% 11.11% { 2816, 262144, 23}, // 44 93 0.12% 12.53% 10.00% - { 3200, 262144, 20}, // 45 81 1.15% 12.54% 13.64% + { 3200, 262144, 20}, // 45 81 1.15% 12.53% 13.64% { 3584, 262144, 18}, // 46 73 0.22% 12.53% 12.00% { 3840, 262144, 17}, // 47 68 0.41% 12.53% 7.14% { 4096, 262144, 16}, // 48 64 0.02% 12.53% 6.67% { 4224, 262144, 15}, // 49 62 0.12% 12.53% 3.12% { 4480, 262144, 14}, // 50 58 0.90% 12.54% 6.06% - { 4736, 262144, 13}, // 51 55 0.66% 12.54% 5.71% + { 4736, 262144, 13}, // 51 55 0.66% 12.53% 5.71% { 5120, 262144, 12}, // 52 51 0.41% 12.53% 8.11% { 5632, 262144, 11}, // 53 46 1.20% 12.54% 10.00% { 6144, 262144, 10}, // 54 42 1.59% 12.54% 9.09% { 6528, 262144, 10}, // 55 40 0.41% 12.53% 6.25% { 7040, 262144, 9}, // 56 37 0.66% 12.54% 7.84% - { 7680, 262144, 8}, // 57 34 0.41% 12.54% 9.09% + { 7680, 262144, 8}, // 57 34 0.41% 12.53% 9.09% { 8192, 262144, 8}, // 58 32 0.02% 12.53% 6.67% { 9344, 262144, 7}, // 59 28 0.22% 12.53% 14.06% { 10368, 262144, 6}, // 60 25 1.15% 12.54% 10.96% @@ -685,7 +685,7 @@ static constexpr SizeClassInfo List[] = { { 87296, 262144, 2}, // 80 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 81 5 0.04% 12.54% 20.09% {112256, 786432, 2}, // 82 7 0.09% 12.54% 7.08% - {131072, 262144, 2}, // 83 2 0.02% 12.54% 16.76% + {131072, 262144, 2}, // 83 2 0.02% 12.53% 16.76% {149760, 786432, 2}, // 84 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 85 3 0.04% 12.54% 16.67% {196608, 786432, 2}, // 86 4 0.01% 12.53% 12.53% diff --git a/tcmalloc/malloc_extension.h b/tcmalloc/malloc_extension.h index a3bf70e7b..06f954a30 100644 --- a/tcmalloc/malloc_extension.h +++ b/tcmalloc/malloc_extension.h @@ -151,7 +151,8 @@ enum class ProfileType { kHeap, // Fragmentation report - kFragmentation, + kFragmentation ABSL_DEPRECATED( + "Fragmentation profiles are no longer collected"), // Sample of objects that were live at a recent peak of total heap usage. The // specifics of when exactly this profile is collected are subject to change. diff --git a/tcmalloc/reuse_relaxed_below_64_size_classes.cc b/tcmalloc/reuse_relaxed_below_64_size_classes.cc index b00ce9eaa..03846e8e3 100644 --- a/tcmalloc/reuse_relaxed_below_64_size_classes.cc +++ b/tcmalloc/reuse_relaxed_below_64_size_classes.cc @@ -78,9 +78,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 18 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 19 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 20 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 21 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 21 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 22 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 23 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 23 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 24 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 25 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 26 4 0.78% 0.42% 14.29% @@ -227,7 +227,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -308,9 +308,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 17 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 18 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 19 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 20 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 20 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 21 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 22 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 22 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 23 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 24 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 25 4 0.78% 0.42% 14.29% @@ -393,7 +393,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -457,7 +457,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/size_classes.cc b/tcmalloc/size_classes.cc index a61a0b0e5..951a012c8 100644 --- a/tcmalloc/size_classes.cc +++ b/tcmalloc/size_classes.cc @@ -76,9 +76,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -161,7 +161,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -225,7 +225,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 262144, 2}, // 43 4 0.02% 12.53% 25.18% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 33.20% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% @@ -305,9 +305,9 @@ static constexpr SizeClassInfo List[] = { { 512, 8192, 32}, // 16 16 0.78% 0.42% 14.29% { 576, 8192, 32}, // 17 14 2.33% 0.42% 12.50% { 704, 8192, 32}, // 18 11 6.20% 0.43% 22.22% - { 896, 8192, 32}, // 19 9 2.33% 0.43% 27.27% + { 896, 8192, 32}, // 19 9 2.33% 0.42% 27.27% { 1024, 8192, 32}, // 20 8 0.78% 0.42% 14.29% - { 1152, 16384, 32}, // 21 14 1.95% 0.43% 12.50% + { 1152, 16384, 32}, // 21 14 1.95% 0.42% 12.50% { 1408, 16384, 32}, // 22 11 5.84% 0.43% 22.22% { 1792, 16384, 32}, // 23 9 1.95% 0.43% 27.27% { 2048, 8192, 32}, // 24 4 0.78% 0.42% 14.29% @@ -390,7 +390,7 @@ static constexpr SizeClassInfo List[] = { { 65536, 65536, 2}, // 40 1 0.10% 0.03% 20.19% { 81920, 163840, 2}, // 41 2 0.04% 4.72% 25.00% { 98304, 98304, 2}, // 42 1 0.07% 0.03% 20.00% - {114688, 229376, 2}, // 43 2 0.03% 6.29% 16.67% + {114688, 229376, 2}, // 43 2 0.03% 6.28% 16.67% {131072, 131072, 2}, // 44 1 0.05% 0.03% 14.29% {163840, 163840, 2}, // 45 1 0.04% 0.03% 25.00% {229376, 229376, 2}, // 46 1 0.03% 0.03% 40.00% @@ -454,7 +454,7 @@ static constexpr SizeClassInfo List[] = { { 74880, 524288, 2}, // 43 7 0.04% 12.53% 14.26% { 87296, 262144, 2}, // 44 3 0.12% 12.54% 16.58% {104832, 524288, 2}, // 45 5 0.04% 12.54% 20.09% - {131072, 262144, 2}, // 46 2 0.02% 12.54% 25.03% + {131072, 262144, 2}, // 46 2 0.02% 12.53% 25.03% {149760, 786432, 2}, // 47 5 4.79% 12.88% 14.26% {174720, 524288, 2}, // 48 3 0.04% 12.54% 16.67% {209664, 1048576, 2}, // 49 5 0.03% 12.54% 20.00% diff --git a/tcmalloc/tcmalloc.cc b/tcmalloc/tcmalloc.cc index 7b2754439..b9b0fa377 100644 --- a/tcmalloc/tcmalloc.cc +++ b/tcmalloc/tcmalloc.cc @@ -119,6 +119,7 @@ #include "tcmalloc/sampler.h" #include "tcmalloc/segv_handler.h" #include "tcmalloc/span.h" +#include "tcmalloc/stack_trace_table.h" #include "tcmalloc/static_vars.h" #include "tcmalloc/stats.h" #include "tcmalloc/tcmalloc_policy.h" @@ -1193,21 +1194,19 @@ alloc_small_sampled_hooks_or_perthread(size_t size, size_t size_class, size_class = ret.size_class; TC_CHECK(ret.is_small); } - void* res; - // If we are here because of sampling, try AllocateFast first. - if (ABSL_PREDICT_TRUE(weight == 0) || - (res = tc_globals.cpu_cache().AllocateFast(size_class)) == nullptr) { + __sized_ptr_t ptr; + if (ABSL_PREDICT_FALSE(weight != 0)) { + ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class); + } else { if (UsePerCpuCache(tc_globals)) { - res = tc_globals.cpu_cache().AllocateSlow(size_class); + ptr.p = tc_globals.cpu_cache().AllocateSlow(size_class); } else { - res = ThreadCache::GetCache()->Allocate(size_class); + ptr.p = ThreadCache::GetCache()->Allocate(size_class); } - if (ABSL_PREDICT_FALSE(res == nullptr)) return policy.handle_oom(size); + ptr.n = tc_globals.sizemap().class_to_size(size_class); } - __sized_ptr_t ptr = {res, tc_globals.sizemap().class_to_size(size_class)}; - if (ABSL_PREDICT_FALSE(weight != 0)) { - ptr = SampleSmallAllocation(tc_globals, policy, size, weight, size_class, - ptr); + if (ABSL_PREDICT_FALSE(ptr.p == nullptr)) { + return policy.handle_oom(size); } if (Policy::invoke_hooks()) { // Size returning tcmallocs call NewHooks with capacity as requested_size.