diff --git a/tcmalloc/BUILD b/tcmalloc/BUILD index 990d12b16..5b5e66dbd 100644 --- a/tcmalloc/BUILD +++ b/tcmalloc/BUILD @@ -206,6 +206,7 @@ create_tcmalloc_libraries( "huge_pages.h", "huge_region.h", "legacy_size_classes.cc", + "lowfrag_size_classes.cc", "metadata_object_allocator.h", "page_allocator.cc", "page_allocator.h", diff --git a/tcmalloc/experiment_config.h b/tcmalloc/experiment_config.h index f3c087a33..3ef2bfd3c 100644 --- a/tcmalloc/experiment_config.h +++ b/tcmalloc/experiment_config.h @@ -31,6 +31,7 @@ enum class Experiment : int { TEST_ONLY_MM_VCPU, // TODO: b/245776120 - Complete experiment. TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING, // TODO: b/328301906 - Complete experiment. TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, // TODO: b/446814339 - Complete experiment. + TEST_ONLY_TCMALLOC_LOWFRAG_SIZECLASSES_V2, TEST_ONLY_TCMALLOC_MADV_COLD_HUGEPAGE, // TODO: b/450368905 - Complete experiment. TEST_ONLY_TCMALLOC_POW2_SIZECLASS, TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, @@ -58,6 +59,7 @@ inline constexpr ExperimentConfig experiments[] = { {Experiment::TEST_ONLY_MM_VCPU, "TEST_ONLY_MM_VCPU"}, {Experiment::TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING, "TEST_ONLY_TCMALLOC_ALWAYS_DISCARDING", /*brittle=*/true}, {Experiment::TEST_ONLY_TCMALLOC_HEAP_PARTITIONING, "TEST_ONLY_TCMALLOC_HEAP_PARTITIONING"}, + {Experiment::TEST_ONLY_TCMALLOC_LOWFRAG_SIZECLASSES_V2, "TEST_ONLY_TCMALLOC_LOWFRAG_SIZECLASSES_V2"}, {Experiment::TEST_ONLY_TCMALLOC_MADV_COLD_HUGEPAGE, "TEST_ONLY_TCMALLOC_MADV_COLD_HUGEPAGE"}, {Experiment::TEST_ONLY_TCMALLOC_POW2_SIZECLASS, "TEST_ONLY_TCMALLOC_POW2_SIZECLASS", /*brittle=*/true}, {Experiment::TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE, "TEST_ONLY_TCMALLOC_SHARDED_TRANSFER_CACHE", /*brittle=*/true}, diff --git a/tcmalloc/global_stats.cc b/tcmalloc/global_stats.cc index c9c76ac7b..a22b839f0 100644 --- a/tcmalloc/global_stats.cc +++ b/tcmalloc/global_stats.cc @@ -276,6 +276,8 @@ static absl::string_view SizeClassConfigurationString( case SizeClassConfiguration::kLegacy: // TODO(b/242710633): remove this opt out. return "SIZE_CLASS_LEGACY"; + case SizeClassConfiguration::kLowFrag: + return "SIZE_CLASS_LOW_FRAG"; case SizeClassConfiguration::kReuse: return "SIZE_CLASS_REUSE"; case SizeClassConfiguration::kReuseRelaxedBelow64: diff --git a/tcmalloc/lowfrag_size_classes.cc b/tcmalloc/lowfrag_size_classes.cc new file mode 100644 index 000000000..60e9fb44e --- /dev/null +++ b/tcmalloc/lowfrag_size_classes.cc @@ -0,0 +1,751 @@ +// Copyright 2019 The TCMalloc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/span.h" +#include "tcmalloc/common.h" +#include "tcmalloc/internal/config.h" +#include "tcmalloc/internal/size_class_info.h" + +GOOGLE_MALLOC_SECTION_BEGIN +namespace tcmalloc { +namespace tcmalloc_internal { + +// Columns in the following tables: +// - bytes: size of the size class +// - span_bytes: number of bytes per span +// - batch: preferred number of objects for transfers between caches +// - class: size class number +// - objs: number of objects per span +// - waste/fixed: fixed per-size-class overhead due to end-of-span fragmentation +// and other factors. For instance, if we have a 96 byte size class, and use +// a single 8KiB page, then we will hold 85 objects per span, and have 32 +// bytes left over. There is also a fixed component of 64 bytes of TCMalloc +// metadata per span. Together, the fixed overhead would be wasted/allocated +// = (32 + 64) / (8192 - 32) ~= 1.2%. +// - waste/sampling: overhead due to heap sampling +// (rounding to page size, proxy object, metadata). +// - inc: increment from the previous size class. This caps the dynamic +// overhead component based on mismatches between the number of bytes +// requested and the number of bytes provided by the size class. Together +// they sum to the total overhead; for instance if you asked for a 50-byte +// allocation that rounds up to a 64-byte size class, the dynamic overhead +// would be 28%, and if waste were 22% it would mean (on average) 25 bytes +// of overhead for allocations of that size. + +// clang-format off +#if defined(__cpp_aligned_new) && __STDCPP_DEFAULT_NEW_ALIGNMENT__ <= 8 +#if TCMALLOC_PAGE_SHIFT == 13 +static_assert(kMaxSize == 262144, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = true, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 8192, 32}, // 1 1024 0.78% 0.42% 0.00% + { 16, 8192, 32}, // 2 512 0.78% 0.42% 100.00% + { 32, 8192, 32}, // 3 256 0.78% 0.42% 100.00% + { 64, 8192, 32}, // 4 128 0.78% 0.42% 100.00% + { 72, 8192, 32}, // 5 113 1.45% 0.42% 12.50% + { 80, 8192, 32}, // 6 102 1.16% 0.42% 11.11% + { 88, 8192, 32}, // 7 93 0.87% 0.42% 10.00% + { 96, 8192, 32}, // 8 85 1.16% 0.42% 9.09% + { 104, 8192, 32}, // 9 78 1.74% 0.42% 8.33% + { 112, 8192, 32}, // 10 73 0.97% 0.42% 7.69% + { 120, 8192, 32}, // 11 68 1.16% 0.42% 7.14% + { 128, 8192, 32}, // 12 64 0.78% 0.42% 6.67% + { 136, 8192, 32}, // 13 60 1.16% 0.42% 6.25% + { 144, 8192, 32}, // 14 56 2.33% 0.42% 5.88% + { 160, 8192, 32}, // 15 51 1.16% 0.42% 11.11% + { 176, 8192, 32}, // 16 46 1.94% 0.42% 10.00% + { 192, 8192, 32}, // 17 42 2.33% 0.42% 9.09% + { 208, 8192, 32}, // 18 39 1.74% 0.42% 8.33% + { 224, 8192, 32}, // 19 36 2.33% 0.42% 7.69% + { 240, 8192, 32}, // 20 34 1.16% 0.42% 7.14% + { 256, 8192, 32}, // 21 32 0.78% 0.42% 6.67% + { 272, 8192, 32}, // 22 30 1.16% 0.42% 6.25% + { 288, 8192, 32}, // 23 28 2.33% 0.42% 5.88% + { 312, 8192, 32}, // 24 26 1.74% 0.42% 8.33% + { 352, 8192, 32}, // 25 23 1.94% 0.42% 12.82% + { 384, 8192, 32}, // 26 21 2.33% 0.42% 9.09% + { 408, 8192, 32}, // 27 20 1.16% 0.42% 6.25% + { 448, 8192, 32}, // 28 18 2.33% 0.42% 9.80% + { 512, 8192, 32}, // 29 16 0.78% 0.42% 14.29% + { 544, 8192, 32}, // 30 15 1.16% 0.42% 6.25% + { 584, 8192, 32}, // 31 14 0.97% 0.42% 7.35% + { 624, 8192, 32}, // 32 13 1.74% 0.42% 6.85% + { 680, 8192, 32}, // 33 12 1.16% 0.42% 8.97% + { 744, 8192, 32}, // 34 11 0.87% 0.42% 9.41% + { 816, 8192, 32}, // 35 10 1.16% 0.42% 9.68% + { 904, 8192, 32}, // 36 9 1.45% 0.42% 10.78% + { 1024, 8192, 32}, // 37 8 0.78% 0.42% 13.27% + { 1152, 8192, 32}, // 38 7 2.33% 0.43% 12.50% + { 1280, 8192, 32}, // 39 6 6.98% 0.43% 11.11% + { 1536, 24576, 32}, // 40 16 0.26% 0.42% 20.00% + { 1792, 16384, 32}, // 41 9 1.95% 0.43% 16.67% + { 2048, 8192, 32}, // 42 4 0.78% 0.42% 14.29% + { 2176, 32768, 30}, // 43 15 0.58% 0.42% 6.25% + { 2432, 24576, 26}, // 44 10 1.30% 0.43% 11.76% + { 2688, 8192, 24}, // 45 3 2.33% 0.43% 10.53% + { 3072, 24576, 21}, // 46 8 0.26% 0.42% 14.29% + { 3456, 24576, 18}, // 47 7 1.82% 0.43% 12.50% + { 3712, 40960, 17}, // 48 11 0.47% 0.43% 7.41% + { 4096, 8192, 16}, // 49 2 0.78% 0.43% 10.34% + { 4736, 57344, 13}, // 50 12 1.00% 0.43% 15.62% + { 5376, 16384, 12}, // 51 3 1.95% 0.43% 13.51% + { 6144, 24576, 10}, // 52 4 0.26% 0.42% 14.29% + { 7168, 57344, 9}, // 53 8 0.11% 0.42% 16.67% + { 8192, 8192, 8}, // 54 1 0.78% 0.03% 14.29% + { 9344, 65536, 7}, // 55 7 0.29% 0.82% 14.06% + { 10880, 32768, 6}, // 56 3 0.58% 0.82% 16.44% + { 12288, 24576, 5}, // 57 2 0.26% 0.82% 12.94% + { 13568, 40960, 4}, // 58 3 0.78% 0.82% 10.42% + { 16384, 16384, 4}, // 59 1 0.39% 0.03% 20.75% + { 17664, 106496, 3}, // 60 6 0.54% 1.21% 7.81% + { 20480, 40960, 3}, // 61 2 0.16% 1.21% 15.94% + { 24576, 24576, 2}, // 62 1 0.26% 0.03% 20.00% + { 26624, 106496, 2}, // 63 4 0.06% 1.60% 8.33% + { 28672, 57344, 2}, // 64 2 0.11% 1.60% 7.69% + { 30720, 122880, 2}, // 65 4 0.05% 1.60% 7.14% + { 32768, 32768, 2}, // 66 1 0.19% 0.03% 6.67% + { 34048, 204800, 2}, // 67 6 0.28% 1.99% 3.91% + { 40960, 40960, 2}, // 68 1 0.16% 0.03% 20.30% + { 49152, 49152, 2}, // 69 1 0.13% 0.03% 20.00% + { 57344, 57344, 2}, // 70 1 0.11% 0.03% 16.67% + { 65536, 65536, 2}, // 71 1 0.10% 0.03% 14.29% + { 68224, 204800, 2}, // 72 3 0.09% 3.55% 4.10% + { 81920, 81920, 2}, // 73 1 0.08% 0.03% 20.08% + { 98304, 98304, 2}, // 74 1 0.07% 0.03% 20.00% + {114688, 114688, 2}, // 75 1 0.06% 0.03% 16.67% + {131072, 131072, 2}, // 76 1 0.05% 0.03% 14.29% + {147456, 147456, 2}, // 77 1 0.04% 0.03% 12.50% + {172032, 172032, 2}, // 78 1 0.04% 0.03% 16.67% + {204800, 204800, 2}, // 79 1 0.03% 0.03% 19.05% + {237568, 237568, 2}, // 80 1 0.03% 0.03% 16.00% + {262144, 262144, 2}, // 81 1 0.02% 0.03% 10.34% +}; +#elif TCMALLOC_PAGE_SHIFT == 15 +static_assert(kMaxSize == 262144, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = true, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 32768, 32}, // 1 4096 0.19% 1.60% 0.00% + { 16, 32768, 32}, // 2 2048 0.19% 1.60% 100.00% + { 32, 32768, 32}, // 3 1024 0.19% 1.60% 100.00% + { 64, 32768, 32}, // 4 512 0.19% 1.60% 100.00% + { 72, 32768, 32}, // 5 455 0.22% 1.60% 12.50% + { 80, 32768, 32}, // 6 409 0.34% 1.60% 11.11% + { 88, 32768, 32}, // 7 372 0.29% 1.60% 10.00% + { 96, 32768, 32}, // 8 341 0.29% 1.60% 9.09% + { 104, 32768, 32}, // 9 315 0.22% 1.60% 8.33% + { 112, 32768, 32}, // 10 292 0.39% 1.60% 7.69% + { 120, 32768, 32}, // 11 273 0.22% 1.60% 7.14% + { 128, 32768, 32}, // 12 256 0.19% 1.60% 6.67% + { 144, 32768, 32}, // 13 227 0.44% 1.60% 12.50% + { 160, 32768, 32}, // 14 204 0.58% 1.60% 11.11% + { 176, 32768, 32}, // 15 186 0.29% 1.60% 10.00% + { 192, 32768, 32}, // 16 170 0.58% 1.60% 9.09% + { 208, 32768, 32}, // 17 157 0.54% 1.60% 8.33% + { 224, 32768, 32}, // 18 146 0.39% 1.60% 7.69% + { 240, 32768, 32}, // 19 136 0.58% 1.60% 7.14% + { 256, 32768, 32}, // 20 128 0.19% 1.60% 6.67% + { 264, 32768, 32}, // 21 124 0.29% 1.60% 3.12% + { 288, 32768, 32}, // 22 113 0.88% 1.60% 9.09% + { 320, 32768, 32}, // 23 102 0.58% 1.60% 11.11% + { 368, 32768, 32}, // 24 89 0.24% 1.60% 15.00% + { 400, 32768, 32}, // 25 81 1.32% 1.60% 8.70% + { 448, 32768, 32}, // 26 73 0.39% 1.60% 12.00% + { 512, 32768, 32}, // 27 64 0.19% 1.60% 14.29% + { 592, 32768, 32}, // 28 55 0.83% 1.60% 15.62% + { 680, 32768, 32}, // 29 48 0.58% 1.60% 14.86% + { 744, 32768, 32}, // 30 44 0.29% 1.60% 9.41% + { 840, 32768, 32}, // 31 39 0.22% 1.60% 12.90% + { 904, 32768, 32}, // 32 36 0.88% 1.60% 7.62% + { 1024, 32768, 32}, // 33 32 0.19% 1.60% 13.27% + { 1152, 32768, 32}, // 34 28 1.75% 1.60% 12.50% + { 1280, 32768, 32}, // 35 25 2.53% 1.60% 11.11% + { 1408, 32768, 32}, // 36 23 1.36% 1.60% 10.00% + { 1536, 32768, 32}, // 37 21 1.75% 1.60% 9.09% + { 1664, 32768, 32}, // 38 19 3.70% 1.60% 8.33% + { 1920, 32768, 32}, // 39 17 0.58% 1.60% 15.38% + { 2048, 32768, 32}, // 40 16 0.19% 1.60% 6.67% + { 2176, 32768, 30}, // 41 15 0.58% 1.60% 6.25% + { 2304, 32768, 28}, // 42 14 1.75% 1.60% 5.88% + { 2688, 32768, 24}, // 43 12 1.75% 1.60% 16.67% + { 2944, 32768, 22}, // 44 11 1.36% 1.60% 9.52% + { 3200, 32768, 20}, // 45 10 2.53% 1.60% 8.70% + { 3584, 32768, 18}, // 46 9 1.75% 1.60% 12.00% + { 4096, 32768, 16}, // 47 8 0.19% 1.60% 14.29% + { 4608, 32768, 14}, // 48 7 1.75% 1.60% 12.50% + { 5376, 32768, 12}, // 49 6 1.75% 1.60% 16.67% + { 6528, 32768, 10}, // 50 5 0.58% 1.60% 21.43% + { 8192, 32768, 8}, // 51 4 0.19% 1.60% 25.49% + { 9344, 65536, 7}, // 52 7 0.29% 1.60% 14.06% + { 10880, 32768, 6}, // 53 3 0.58% 1.60% 16.44% + { 13568, 163840, 4}, // 54 12 0.66% 1.60% 24.71% + { 16384, 32768, 4}, // 55 2 0.19% 1.60% 20.75% + { 18688, 131072, 3}, // 56 7 0.24% 1.60% 14.06% + { 21760, 65536, 3}, // 57 3 0.49% 1.60% 16.44% + { 26112, 131072, 2}, // 58 5 0.44% 1.60% 20.00% + { 32768, 32768, 2}, // 59 1 0.19% 0.03% 25.49% + { 35200, 458752, 2}, // 60 13 0.27% 3.16% 7.42% + { 40960, 163840, 2}, // 61 4 0.04% 3.16% 16.36% + { 49152, 98304, 2}, // 62 2 0.07% 3.16% 20.00% + { 54528, 163840, 2}, // 63 3 0.20% 3.16% 10.94% + { 65536, 65536, 2}, // 64 1 0.10% 0.03% 20.19% + { 72064, 360448, 2}, // 65 5 0.05% 4.72% 9.96% + { 81920, 163840, 2}, // 66 2 0.04% 4.72% 13.68% + { 98304, 98304, 2}, // 67 1 0.07% 0.03% 20.00% + {114688, 229376, 2}, // 68 2 0.03% 6.29% 16.67% + {131072, 131072, 2}, // 69 1 0.05% 0.03% 14.29% + {163840, 163840, 2}, // 70 1 0.04% 0.03% 25.00% + {196608, 196608, 2}, // 71 1 0.03% 0.03% 20.00% + {212992, 425984, 2}, // 72 2 0.02% 10.97% 8.33% + {262144, 262144, 2}, // 73 1 0.02% 0.03% 23.08% +}; +#elif TCMALLOC_PAGE_SHIFT == 18 +static_assert(kMaxSize == 262144, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = true, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 262144, 32}, // 1 32768 0.02% 12.53% 0.00% + { 16, 262144, 32}, // 2 16384 0.02% 12.53% 100.00% + { 32, 262144, 32}, // 3 8192 0.02% 12.53% 100.00% + { 64, 262144, 32}, // 4 4096 0.02% 12.53% 100.00% + { 72, 262144, 32}, // 5 3640 0.05% 12.53% 12.50% + { 80, 262144, 32}, // 6 3276 0.05% 12.53% 11.11% + { 88, 262144, 32}, // 7 2978 0.05% 12.53% 10.00% + { 96, 262144, 32}, // 8 2730 0.05% 12.53% 9.09% + { 104, 262144, 32}, // 9 2520 0.05% 12.53% 8.33% + { 112, 262144, 32}, // 10 2340 0.05% 12.53% 7.69% + { 128, 262144, 32}, // 11 2048 0.02% 12.53% 14.29% + { 144, 262144, 32}, // 12 1820 0.05% 12.53% 12.50% + { 160, 262144, 32}, // 13 1638 0.05% 12.53% 11.11% + { 176, 262144, 32}, // 14 1489 0.05% 12.53% 10.00% + { 192, 262144, 32}, // 15 1365 0.05% 12.53% 9.09% + { 224, 262144, 32}, // 16 1170 0.05% 12.53% 16.67% + { 240, 262144, 32}, // 17 1092 0.05% 12.53% 7.14% + { 256, 262144, 32}, // 18 1024 0.02% 12.53% 6.67% + { 304, 262144, 32}, // 19 862 0.06% 12.53% 18.75% + { 344, 262144, 32}, // 20 762 0.03% 12.53% 13.16% + { 384, 262144, 32}, // 21 682 0.12% 12.53% 11.63% + { 448, 262144, 32}, // 22 585 0.05% 12.53% 16.67% + { 480, 262144, 32}, // 23 546 0.05% 12.53% 7.14% + { 520, 262144, 32}, // 24 504 0.05% 12.53% 8.33% + { 576, 262144, 32}, // 25 455 0.05% 12.53% 10.77% + { 648, 262144, 32}, // 26 404 0.16% 12.53% 12.50% + { 728, 262144, 32}, // 27 360 0.05% 12.53% 12.35% + { 848, 262144, 32}, // 28 309 0.07% 12.53% 16.48% + { 1024, 262144, 32}, // 29 256 0.02% 12.53% 20.75% + { 1152, 262144, 32}, // 30 227 0.27% 12.53% 12.50% + { 1280, 262144, 32}, // 31 204 0.41% 12.53% 11.11% + { 1408, 262144, 32}, // 32 186 0.12% 12.53% 10.00% + { 1664, 262144, 32}, // 33 157 0.37% 12.53% 18.18% + { 1920, 262144, 32}, // 34 136 0.41% 12.53% 15.38% + { 2048, 262144, 32}, // 35 128 0.02% 12.53% 6.67% + { 2304, 262144, 28}, // 36 113 0.71% 12.53% 12.50% + { 2688, 262144, 24}, // 37 97 0.56% 12.53% 16.67% + { 3200, 262144, 20}, // 38 81 1.15% 12.54% 19.05% + { 3584, 262144, 18}, // 39 73 0.22% 12.53% 12.00% + { 4096, 262144, 16}, // 40 64 0.02% 12.53% 14.29% + { 4224, 262144, 15}, // 41 62 0.12% 12.53% 3.12% + { 4608, 262144, 14}, // 42 56 1.59% 12.54% 9.09% + { 5120, 262144, 12}, // 43 51 0.41% 12.53% 11.11% + { 5632, 262144, 11}, // 44 46 1.20% 12.54% 10.00% + { 6144, 262144, 10}, // 45 42 1.59% 12.54% 9.09% + { 6656, 262144, 9}, // 46 39 1.00% 12.54% 8.33% + { 7424, 262144, 8}, // 47 35 0.90% 12.54% 11.54% + { 8192, 262144, 8}, // 48 32 0.02% 12.53% 10.34% + { 9344, 262144, 7}, // 49 28 0.22% 12.53% 14.06% + { 11392, 262144, 5}, // 50 23 0.07% 12.53% 21.92% + { 13696, 262144, 4}, // 51 19 0.76% 12.54% 20.22% + { 16384, 262144, 4}, // 52 16 0.02% 12.53% 19.63% + { 18688, 262144, 3}, // 53 14 0.22% 12.54% 14.06% + { 21760, 262144, 3}, // 54 12 0.41% 12.54% 16.44% + { 23808, 262144, 2}, // 55 11 0.12% 12.53% 9.41% + { 26112, 262144, 2}, // 56 10 0.41% 12.54% 9.68% + { 29056, 262144, 2}, // 57 9 0.27% 12.54% 11.27% + { 32768, 262144, 2}, // 58 8 0.02% 12.53% 12.78% + { 34944, 524288, 2}, // 59 15 0.04% 12.53% 6.64% + { 37376, 262144, 2}, // 60 7 0.22% 12.54% 6.96% + { 40320, 524288, 2}, // 61 13 0.04% 12.53% 7.88% + { 43648, 262144, 2}, // 62 6 0.12% 12.54% 8.25% + { 47616, 524288, 2}, // 63 11 0.11% 12.54% 9.09% + { 52352, 262144, 2}, // 64 5 0.17% 12.54% 9.95% + { 58240, 524288, 2}, // 65 9 0.04% 12.53% 11.25% + { 65536, 262144, 2}, // 66 4 0.02% 12.53% 12.53% + { 69888, 1048576, 2}, // 67 15 0.03% 12.53% 6.64% + { 78592, 786432, 2}, // 68 10 0.07% 12.54% 12.45% + { 87296, 262144, 2}, // 69 3 0.12% 12.54% 11.07% + { 93568, 1310720, 2}, // 70 14 0.06% 12.54% 7.18% + { 98304, 786432, 2}, // 71 8 0.01% 12.53% 5.06% + {100736, 1310720, 2}, // 72 13 0.09% 12.54% 2.47% + {104832, 524288, 2}, // 73 5 0.04% 12.54% 4.07% + {109184, 1310720, 2}, // 74 12 0.04% 12.54% 4.15% + {116480, 1048576, 2}, // 75 9 0.03% 12.54% 6.68% + {131072, 262144, 2}, // 76 2 0.02% 12.54% 12.53% + {141056, 1835008, 2}, // 77 13 0.07% 12.54% 7.62% + {149760, 1048576, 2}, // 78 7 0.03% 12.54% 6.17% + {157184, 786432, 2}, // 79 5 0.07% 12.54% 4.96% + {163840, 1310720, 2}, // 80 8 0.00% 12.53% 4.23% + {174720, 524288, 2}, // 81 3 0.04% 12.54% 6.64% + {196608, 786432, 2}, // 82 4 0.01% 12.53% 12.53% + {218368, 1310720, 2}, // 83 6 0.04% 12.54% 11.07% + {262144, 262144, 2}, // 84 1 0.02% 0.03% 20.05% +}; +#elif TCMALLOC_PAGE_SHIFT == 12 +static_assert(kMaxSize == 8192, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = false, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 4096, 32}, // 1 512 1.54% 0.23% 0.00% + { 16, 4096, 32}, // 2 256 1.54% 0.23% 100.00% + { 32, 4096, 32}, // 3 128 1.54% 0.23% 100.00% + { 64, 4096, 32}, // 4 64 1.54% 0.23% 100.00% + { 72, 4096, 32}, // 5 56 3.08% 0.23% 12.50% + { 80, 4096, 32}, // 6 51 1.92% 0.23% 11.11% + { 88, 4096, 32}, // 7 46 2.69% 0.23% 10.00% + { 96, 4096, 32}, // 8 42 3.08% 0.23% 9.09% + { 104, 4096, 32}, // 9 39 2.50% 0.23% 8.33% + { 120, 4096, 32}, // 10 34 1.92% 0.23% 15.38% + { 128, 4096, 32}, // 11 32 1.54% 0.23% 6.67% + { 136, 4096, 32}, // 12 30 1.92% 0.23% 6.25% + { 144, 4096, 32}, // 13 28 3.08% 0.23% 5.88% + { 160, 4096, 32}, // 14 25 3.85% 0.23% 11.11% + { 176, 4096, 32}, // 15 23 2.69% 0.23% 10.00% + { 192, 4096, 32}, // 16 21 3.08% 0.23% 9.09% + { 208, 4096, 32}, // 17 19 5.00% 0.23% 8.33% + { 240, 4096, 32}, // 18 17 1.92% 0.23% 15.38% + { 256, 4096, 32}, // 19 16 1.54% 0.23% 6.67% + { 272, 4096, 32}, // 20 15 1.92% 0.23% 6.25% + { 312, 4096, 32}, // 21 13 2.50% 0.23% 14.71% + { 368, 4096, 32}, // 22 11 2.69% 0.23% 17.95% + { 448, 4096, 32}, // 23 9 3.08% 0.23% 21.74% + { 512, 4096, 32}, // 24 8 1.54% 0.23% 14.29% + { 584, 4096, 32}, // 25 7 1.73% 0.23% 14.06% + { 680, 4096, 32}, // 26 6 1.92% 0.23% 16.44% + { 768, 12288, 32}, // 27 16 0.52% 0.23% 12.94% + { 904, 8192, 32}, // 28 9 1.45% 0.23% 17.71% + { 1024, 4096, 32}, // 29 4 1.54% 0.23% 13.27% + { 1280, 20480, 32}, // 30 16 0.31% 0.23% 25.00% + { 1408, 20480, 32}, // 31 14 4.05% 0.23% 10.00% + { 1664, 20480, 32}, // 32 12 2.80% 0.23% 18.18% + { 2048, 4096, 32}, // 33 2 1.54% 0.23% 23.08% + { 2432, 12288, 26}, // 34 5 1.55% 0.23% 18.75% + { 3072, 12288, 21}, // 35 4 0.52% 0.23% 26.32% + { 4096, 4096, 16}, // 36 1 1.54% 0.03% 33.33% + { 4224, 69632, 15}, // 37 16 3.03% 0.43% 3.12% + { 4736, 28672, 13}, // 38 6 1.11% 0.43% 12.12% + { 6144, 12288, 10}, // 39 2 0.52% 0.43% 29.73% + { 7168, 28672, 9}, // 40 4 0.22% 0.42% 16.67% + { 8192, 8192, 8}, // 41 1 0.78% 0.03% 14.29% +}; +#else +#error "Unsupported TCMALLOC_PAGE_SHIFT value!" +#endif +#else +#if TCMALLOC_PAGE_SHIFT == 13 +static_assert(kMaxSize == 262144, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = true, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 8192, 32}, // 1 1024 0.78% 0.42% 0.00% + { 16, 8192, 32}, // 2 512 0.78% 0.42% 100.00% + { 32, 8192, 32}, // 3 256 0.78% 0.42% 100.00% + { 64, 8192, 32}, // 4 128 0.78% 0.42% 100.00% + { 80, 8192, 32}, // 5 102 1.16% 0.42% 25.00% + { 96, 8192, 32}, // 6 85 1.16% 0.42% 20.00% + { 112, 8192, 32}, // 7 73 0.97% 0.42% 16.67% + { 128, 8192, 32}, // 8 64 0.78% 0.42% 14.29% + { 144, 8192, 32}, // 9 56 2.33% 0.42% 12.50% + { 160, 8192, 32}, // 10 51 1.16% 0.42% 11.11% + { 176, 8192, 32}, // 11 46 1.94% 0.42% 10.00% + { 192, 8192, 32}, // 12 42 2.33% 0.42% 9.09% + { 208, 8192, 32}, // 13 39 1.74% 0.42% 8.33% + { 224, 8192, 32}, // 14 36 2.33% 0.42% 7.69% + { 240, 8192, 32}, // 15 34 1.16% 0.42% 7.14% + { 256, 8192, 32}, // 16 32 0.78% 0.42% 6.67% + { 272, 8192, 32}, // 17 30 1.16% 0.42% 6.25% + { 288, 8192, 32}, // 18 28 2.33% 0.42% 5.88% + { 320, 8192, 32}, // 19 25 3.10% 0.42% 11.11% + { 352, 8192, 32}, // 20 23 1.94% 0.42% 10.00% + { 368, 8192, 32}, // 21 22 1.94% 0.42% 4.55% + { 384, 8192, 32}, // 22 21 2.33% 0.42% 4.35% + { 416, 8192, 32}, // 23 19 4.26% 0.43% 8.33% + { 448, 8192, 32}, // 24 18 2.33% 0.42% 7.69% + { 512, 8192, 32}, // 25 16 0.78% 0.42% 14.29% + { 544, 8192, 32}, // 26 15 1.16% 0.42% 6.25% + { 576, 8192, 32}, // 27 14 2.33% 0.42% 5.88% + { 624, 8192, 32}, // 28 13 1.74% 0.42% 8.33% + { 672, 8192, 32}, // 29 12 2.33% 0.42% 7.69% + { 736, 8192, 32}, // 30 11 1.94% 0.42% 9.52% + { 816, 8192, 32}, // 31 10 1.16% 0.42% 10.87% + { 896, 8192, 32}, // 32 9 2.33% 0.43% 9.80% + { 1024, 8192, 32}, // 33 8 0.78% 0.42% 14.29% + { 1152, 8192, 32}, // 34 7 2.33% 0.43% 12.50% + { 1280, 8192, 32}, // 35 6 6.98% 0.43% 11.11% + { 1408, 16384, 32}, // 36 11 5.84% 0.43% 10.00% + { 1536, 24576, 32}, // 37 16 0.26% 0.42% 9.09% + { 1792, 16384, 32}, // 38 9 1.95% 0.43% 16.67% + { 2048, 8192, 32}, // 39 4 0.78% 0.42% 14.29% + { 2176, 32768, 30}, // 40 15 0.58% 0.42% 6.25% + { 2432, 24576, 26}, // 41 10 1.30% 0.43% 11.76% + { 2688, 8192, 24}, // 42 3 2.33% 0.43% 10.53% + { 3072, 24576, 21}, // 43 8 0.26% 0.42% 14.29% + { 3456, 24576, 18}, // 44 7 1.82% 0.43% 12.50% + { 3712, 40960, 17}, // 45 11 0.47% 0.43% 7.41% + { 4096, 8192, 16}, // 46 2 0.78% 0.43% 10.34% + { 4352, 57344, 15}, // 47 13 1.45% 0.43% 6.25% + { 4864, 24576, 13}, // 48 5 1.30% 0.43% 11.76% + { 5376, 16384, 12}, // 49 3 1.95% 0.43% 10.53% + { 6144, 24576, 10}, // 50 4 0.26% 0.42% 14.29% + { 7168, 57344, 9}, // 51 8 0.11% 0.42% 16.67% + { 8192, 8192, 8}, // 52 1 0.78% 0.03% 14.29% + { 9344, 65536, 7}, // 53 7 0.29% 0.82% 14.06% + { 10880, 32768, 6}, // 54 3 0.58% 0.82% 16.44% + { 12288, 24576, 5}, // 55 2 0.26% 0.82% 12.94% + { 13568, 40960, 4}, // 56 3 0.78% 0.82% 10.42% + { 16384, 16384, 4}, // 57 1 0.39% 0.03% 20.75% + { 17024, 188416, 3}, // 58 11 0.65% 1.21% 3.91% + { 19072, 57344, 3}, // 59 3 0.33% 1.21% 12.03% + { 21760, 65536, 3}, // 60 3 0.49% 1.21% 14.09% + { 24576, 24576, 2}, // 61 1 0.26% 0.03% 12.94% + { 26624, 106496, 2}, // 62 4 0.06% 1.60% 8.33% + { 28672, 57344, 2}, // 63 2 0.11% 1.60% 7.69% + { 29952, 90112, 2}, // 64 3 0.35% 1.60% 4.46% + { 30720, 122880, 2}, // 65 4 0.05% 1.60% 2.56% + { 32768, 32768, 2}, // 66 1 0.19% 0.03% 6.67% + { 34048, 204800, 2}, // 67 6 0.28% 1.99% 3.91% + { 36864, 73728, 2}, // 68 2 0.09% 1.99% 8.27% + { 40960, 40960, 2}, // 69 1 0.16% 0.03% 11.11% + { 49152, 49152, 2}, // 70 1 0.13% 0.03% 20.00% + { 57344, 57344, 2}, // 71 1 0.11% 0.03% 16.67% + { 65536, 65536, 2}, // 72 1 0.10% 0.03% 14.29% + { 68224, 204800, 2}, // 73 3 0.09% 3.55% 4.10% + { 73728, 73728, 2}, // 74 1 0.09% 0.03% 8.07% + { 81920, 81920, 2}, // 75 1 0.08% 0.03% 11.11% + { 98304, 98304, 2}, // 76 1 0.07% 0.03% 20.00% + {114688, 114688, 2}, // 77 1 0.06% 0.03% 16.67% + {131072, 131072, 2}, // 78 1 0.05% 0.03% 14.29% + {139264, 139264, 2}, // 79 1 0.05% 0.03% 6.25% + {155648, 155648, 2}, // 80 1 0.04% 0.03% 11.76% + {172032, 172032, 2}, // 81 1 0.04% 0.03% 10.53% + {204800, 204800, 2}, // 82 1 0.03% 0.03% 19.05% + {237568, 237568, 2}, // 83 1 0.03% 0.03% 16.00% + {262144, 262144, 2}, // 84 1 0.02% 0.03% 10.34% +}; +#elif TCMALLOC_PAGE_SHIFT == 15 +static_assert(kMaxSize == 262144, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = true, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 32768, 32}, // 1 4096 0.19% 1.60% 0.00% + { 16, 32768, 32}, // 2 2048 0.19% 1.60% 100.00% + { 32, 32768, 32}, // 3 1024 0.19% 1.60% 100.00% + { 64, 32768, 32}, // 4 512 0.19% 1.60% 100.00% + { 80, 32768, 32}, // 5 409 0.34% 1.60% 25.00% + { 96, 32768, 32}, // 6 341 0.29% 1.60% 20.00% + { 112, 32768, 32}, // 7 292 0.39% 1.60% 16.67% + { 128, 32768, 32}, // 8 256 0.19% 1.60% 14.29% + { 144, 32768, 32}, // 9 227 0.44% 1.60% 12.50% + { 160, 32768, 32}, // 10 204 0.58% 1.60% 11.11% + { 176, 32768, 32}, // 11 186 0.29% 1.60% 10.00% + { 192, 32768, 32}, // 12 170 0.58% 1.60% 9.09% + { 208, 32768, 32}, // 13 157 0.54% 1.60% 8.33% + { 224, 32768, 32}, // 14 146 0.39% 1.60% 7.69% + { 240, 32768, 32}, // 15 136 0.58% 1.60% 7.14% + { 256, 32768, 32}, // 16 128 0.19% 1.60% 6.67% + { 272, 32768, 32}, // 17 120 0.58% 1.60% 6.25% + { 288, 32768, 32}, // 18 113 0.88% 1.60% 5.88% + { 320, 32768, 32}, // 19 102 0.58% 1.60% 11.11% + { 368, 32768, 32}, // 20 89 0.24% 1.60% 15.00% + { 400, 32768, 32}, // 21 81 1.32% 1.60% 8.70% + { 448, 32768, 32}, // 22 73 0.39% 1.60% 12.00% + { 512, 32768, 32}, // 23 64 0.19% 1.60% 14.29% + { 592, 32768, 32}, // 24 55 0.83% 1.60% 15.62% + { 640, 32768, 32}, // 25 51 0.58% 1.60% 8.11% + { 704, 32768, 32}, // 26 46 1.36% 1.60% 10.00% + { 768, 32768, 32}, // 27 42 1.75% 1.60% 9.09% + { 832, 32768, 32}, // 28 39 1.17% 1.60% 8.33% + { 896, 32768, 32}, // 29 36 1.75% 1.60% 7.69% + { 1024, 32768, 32}, // 30 32 0.19% 1.60% 14.29% + { 1152, 32768, 32}, // 31 28 1.75% 1.60% 12.50% + { 1280, 32768, 32}, // 32 25 2.53% 1.60% 11.11% + { 1408, 32768, 32}, // 33 23 1.36% 1.60% 10.00% + { 1536, 32768, 32}, // 34 21 1.75% 1.60% 9.09% + { 1664, 32768, 32}, // 35 19 3.70% 1.60% 8.33% + { 1920, 32768, 32}, // 36 17 0.58% 1.60% 15.38% + { 2048, 32768, 32}, // 37 16 0.19% 1.60% 6.67% + { 2176, 32768, 30}, // 38 15 0.58% 1.60% 6.25% + { 2304, 32768, 28}, // 39 14 1.75% 1.60% 5.88% + { 2688, 32768, 24}, // 40 12 1.75% 1.60% 16.67% + { 2944, 32768, 22}, // 41 11 1.36% 1.60% 9.52% + { 3200, 32768, 20}, // 42 10 2.53% 1.60% 8.70% + { 3584, 32768, 18}, // 43 9 1.75% 1.60% 12.00% + { 4096, 32768, 16}, // 44 8 0.19% 1.60% 14.29% + { 4608, 32768, 14}, // 45 7 1.75% 1.60% 12.50% + { 4992, 65536, 13}, // 46 13 1.07% 1.60% 8.33% + { 5376, 32768, 12}, // 47 6 1.75% 1.60% 7.69% + { 6144, 98304, 10}, // 48 16 0.07% 1.60% 14.29% + { 6528, 32768, 10}, // 49 5 0.58% 1.60% 6.25% + { 7168, 65536, 9}, // 50 9 1.66% 1.60% 9.80% + { 8192, 32768, 8}, // 51 4 0.19% 1.60% 14.29% + { 9344, 65536, 7}, // 52 7 0.29% 1.60% 14.06% + { 10880, 32768, 6}, // 53 3 0.58% 1.60% 16.44% + { 12288, 98304, 5}, // 54 8 0.07% 1.60% 12.94% + { 13952, 98304, 4}, // 55 7 0.72% 1.60% 13.54% + { 16384, 32768, 4}, // 56 2 0.19% 1.60% 17.43% + { 18688, 131072, 3}, // 57 7 0.24% 1.60% 14.06% + { 21760, 65536, 3}, // 58 3 0.49% 1.60% 16.44% + { 24576, 98304, 2}, // 59 4 0.07% 1.60% 12.94% + { 28032, 196608, 2}, // 60 7 0.23% 1.60% 14.06% + { 32768, 32768, 2}, // 61 1 0.19% 0.03% 16.89% + { 35200, 458752, 2}, // 62 13 0.27% 3.16% 7.42% + { 40960, 163840, 2}, // 63 4 0.04% 3.16% 16.36% + { 49152, 98304, 2}, // 64 2 0.07% 3.16% 20.00% + { 54528, 163840, 2}, // 65 3 0.20% 3.16% 10.94% + { 65536, 65536, 2}, // 66 1 0.10% 0.03% 20.19% + { 72064, 360448, 2}, // 67 5 0.05% 4.72% 9.96% + { 81920, 163840, 2}, // 68 2 0.04% 4.72% 13.68% + { 98304, 98304, 2}, // 69 1 0.07% 0.03% 20.00% + {114688, 229376, 2}, // 70 2 0.03% 6.29% 16.67% + {131072, 131072, 2}, // 71 1 0.05% 0.03% 14.29% + {141952, 425984, 2}, // 72 3 0.05% 7.85% 8.30% + {163840, 163840, 2}, // 73 1 0.04% 0.03% 15.42% + {196608, 196608, 2}, // 74 1 0.03% 0.03% 20.00% + {212992, 425984, 2}, // 75 2 0.02% 10.97% 8.33% + {262144, 262144, 2}, // 76 1 0.02% 0.03% 23.08% +}; +#elif TCMALLOC_PAGE_SHIFT == 18 +static_assert(kMaxSize == 262144, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = true, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 262144, 32}, // 1 32768 0.02% 12.53% 0.00% + { 16, 262144, 32}, // 2 16384 0.02% 12.53% 100.00% + { 32, 262144, 32}, // 3 8192 0.02% 12.53% 100.00% + { 64, 262144, 32}, // 4 4096 0.02% 12.53% 100.00% + { 80, 262144, 32}, // 5 3276 0.05% 12.53% 25.00% + { 96, 262144, 32}, // 6 2730 0.05% 12.53% 20.00% + { 112, 262144, 32}, // 7 2340 0.05% 12.53% 16.67% + { 128, 262144, 32}, // 8 2048 0.02% 12.53% 14.29% + { 144, 262144, 32}, // 9 1820 0.05% 12.53% 12.50% + { 160, 262144, 32}, // 10 1638 0.05% 12.53% 11.11% + { 176, 262144, 32}, // 11 1489 0.05% 12.53% 10.00% + { 192, 262144, 32}, // 12 1365 0.05% 12.53% 9.09% + { 208, 262144, 32}, // 13 1260 0.05% 12.53% 8.33% + { 224, 262144, 32}, // 14 1170 0.05% 12.53% 7.69% + { 240, 262144, 32}, // 15 1092 0.05% 12.53% 7.14% + { 256, 262144, 32}, // 16 1024 0.02% 12.53% 6.67% + { 288, 262144, 32}, // 17 910 0.05% 12.53% 12.50% + { 304, 262144, 32}, // 18 862 0.06% 12.53% 5.56% + { 352, 262144, 32}, // 19 744 0.12% 12.53% 15.79% + { 384, 262144, 32}, // 20 682 0.12% 12.53% 9.09% + { 448, 262144, 32}, // 21 585 0.05% 12.53% 16.67% + { 480, 262144, 32}, // 22 546 0.05% 12.53% 7.14% + { 512, 262144, 32}, // 23 512 0.02% 12.53% 6.67% + { 576, 262144, 32}, // 24 455 0.05% 12.53% 12.50% + { 656, 262144, 32}, // 25 399 0.18% 12.53% 13.89% + { 736, 262144, 32}, // 26 356 0.07% 12.53% 12.20% + { 848, 262144, 32}, // 27 309 0.07% 12.53% 15.22% + { 1024, 262144, 32}, // 28 256 0.02% 12.53% 20.75% + { 1152, 262144, 32}, // 29 227 0.27% 12.53% 12.50% + { 1280, 262144, 32}, // 30 204 0.41% 12.53% 11.11% + { 1408, 262144, 32}, // 31 186 0.12% 12.53% 10.00% + { 1536, 262144, 32}, // 32 170 0.41% 12.53% 9.09% + { 1664, 262144, 32}, // 33 157 0.37% 12.53% 8.33% + { 1792, 262144, 32}, // 34 146 0.22% 12.53% 7.69% + { 1920, 262144, 32}, // 35 136 0.41% 12.53% 7.14% + { 2048, 262144, 32}, // 36 128 0.02% 12.53% 6.67% + { 2304, 262144, 28}, // 37 113 0.71% 12.53% 12.50% + { 2688, 262144, 24}, // 38 97 0.56% 12.53% 16.67% + { 3200, 262144, 20}, // 39 81 1.15% 12.54% 19.05% + { 3584, 262144, 18}, // 40 73 0.22% 12.53% 12.00% + { 4096, 262144, 16}, // 41 64 0.02% 12.53% 14.29% + { 4224, 262144, 15}, // 42 62 0.12% 12.53% 3.12% + { 4608, 262144, 14}, // 43 56 1.59% 12.54% 9.09% + { 5120, 262144, 12}, // 44 51 0.41% 12.53% 11.11% + { 5632, 262144, 11}, // 45 46 1.20% 12.54% 10.00% + { 6144, 262144, 10}, // 46 42 1.59% 12.54% 9.09% + { 6656, 262144, 9}, // 47 39 1.00% 12.54% 8.33% + { 7424, 262144, 8}, // 48 35 0.90% 12.54% 11.54% + { 8192, 262144, 8}, // 49 32 0.02% 12.53% 10.34% + { 9344, 262144, 7}, // 50 28 0.22% 12.53% 14.06% + { 11392, 262144, 5}, // 51 23 0.07% 12.53% 21.92% + { 13696, 262144, 4}, // 52 19 0.76% 12.54% 20.22% + { 16384, 262144, 4}, // 53 16 0.02% 12.53% 19.63% + { 18688, 262144, 3}, // 54 14 0.22% 12.54% 14.06% + { 21760, 262144, 3}, // 55 12 0.41% 12.54% 16.44% + { 23808, 262144, 2}, // 56 11 0.12% 12.53% 9.41% + { 26112, 262144, 2}, // 57 10 0.41% 12.54% 9.68% + { 29056, 262144, 2}, // 58 9 0.27% 12.54% 11.27% + { 32768, 262144, 2}, // 59 8 0.02% 12.53% 12.78% + { 34944, 524288, 2}, // 60 15 0.04% 12.53% 6.64% + { 37376, 262144, 2}, // 61 7 0.22% 12.54% 6.96% + { 40320, 524288, 2}, // 62 13 0.04% 12.53% 7.88% + { 43648, 262144, 2}, // 63 6 0.12% 12.54% 8.25% + { 47616, 524288, 2}, // 64 11 0.11% 12.54% 9.09% + { 52352, 262144, 2}, // 65 5 0.17% 12.54% 9.95% + { 58240, 524288, 2}, // 66 9 0.04% 12.53% 11.25% + { 65536, 262144, 2}, // 67 4 0.02% 12.53% 12.53% + { 69888, 1048576, 2}, // 68 15 0.03% 12.53% 6.64% + { 74880, 524288, 2}, // 69 7 0.04% 12.53% 7.14% + { 80640, 1048576, 2}, // 70 13 0.03% 12.53% 7.69% + { 87296, 262144, 2}, // 71 3 0.12% 12.54% 8.25% + { 93568, 1310720, 2}, // 72 14 0.06% 12.54% 7.18% + { 98304, 786432, 2}, // 73 8 0.01% 12.53% 5.06% + {100736, 1310720, 2}, // 74 13 0.09% 12.54% 2.47% + {104832, 524288, 2}, // 75 5 0.04% 12.54% 4.07% + {109184, 1310720, 2}, // 76 12 0.04% 12.54% 4.15% + {112256, 786432, 2}, // 77 7 0.09% 12.54% 2.81% + {119040, 1310720, 2}, // 78 11 0.10% 12.54% 6.04% + {131072, 262144, 2}, // 79 2 0.02% 12.54% 10.11% + {141056, 1835008, 2}, // 80 13 0.07% 12.54% 7.62% + {149760, 1048576, 2}, // 81 7 0.03% 12.54% 6.17% + {157184, 786432, 2}, // 82 5 0.07% 12.54% 4.96% + {163840, 1310720, 2}, // 83 8 0.00% 12.53% 4.23% + {174720, 524288, 2}, // 84 3 0.04% 12.54% 6.64% + {196608, 786432, 2}, // 85 4 0.01% 12.53% 12.53% + {218368, 1310720, 2}, // 86 6 0.04% 12.54% 11.07% + {262144, 262144, 2}, // 87 1 0.02% 0.03% 20.05% +}; +#elif TCMALLOC_PAGE_SHIFT == 12 +static_assert(kMaxSize == 8192, "kMaxSize mismatch"); +static constexpr SizeClassAssumptions Assumptions{ + .has_expanded_classes = false, + .span_size = 64, + .sampling_interval = 2097152, + .large_size = 1024, + .large_size_alignment = 128, +}; +static constexpr SizeClassInfo List[] = { +// | waste | +// bytes span_bytes batch class objs | fixed sampling | inc + { 0, 0, 0}, // 0 0 0.00% 0.00% 0.00% + { 8, 4096, 32}, // 1 512 1.54% 0.23% 0.00% + { 16, 4096, 32}, // 2 256 1.54% 0.23% 100.00% + { 32, 4096, 32}, // 3 128 1.54% 0.23% 100.00% + { 64, 4096, 32}, // 4 64 1.54% 0.23% 100.00% + { 80, 4096, 32}, // 5 51 1.92% 0.23% 25.00% + { 96, 4096, 32}, // 6 42 3.08% 0.23% 20.00% + { 112, 4096, 32}, // 7 36 3.08% 0.23% 16.67% + { 128, 4096, 32}, // 8 32 1.54% 0.23% 14.29% + { 144, 4096, 32}, // 9 28 3.08% 0.23% 12.50% + { 160, 4096, 32}, // 10 25 3.85% 0.23% 11.11% + { 176, 4096, 32}, // 11 23 2.69% 0.23% 10.00% + { 192, 4096, 32}, // 12 21 3.08% 0.23% 9.09% + { 208, 4096, 32}, // 13 19 5.00% 0.23% 8.33% + { 224, 4096, 32}, // 14 18 3.08% 0.23% 7.69% + { 240, 4096, 32}, // 15 17 1.92% 0.23% 7.14% + { 256, 4096, 32}, // 16 16 1.54% 0.23% 6.67% + { 272, 4096, 32}, // 17 15 1.92% 0.23% 6.25% + { 288, 4096, 32}, // 18 14 3.08% 0.23% 5.88% + { 336, 4096, 32}, // 19 12 3.08% 0.23% 16.67% + { 368, 4096, 32}, // 20 11 2.69% 0.23% 9.52% + { 448, 4096, 32}, // 21 9 3.08% 0.23% 21.74% + { 512, 4096, 32}, // 22 8 1.54% 0.23% 14.29% + { 544, 8192, 32}, // 23 15 1.16% 0.23% 6.25% + { 576, 4096, 32}, // 24 7 3.08% 0.23% 5.88% + { 672, 4096, 32}, // 25 6 3.08% 0.23% 16.67% + { 768, 12288, 32}, // 26 16 0.52% 0.23% 14.29% + { 896, 8192, 32}, // 27 9 2.33% 0.23% 16.67% + { 1024, 4096, 32}, // 28 4 1.54% 0.23% 14.29% + { 1152, 8192, 32}, // 29 7 2.33% 0.23% 12.50% + { 1280, 20480, 32}, // 30 16 0.31% 0.23% 11.11% + { 1408, 20480, 32}, // 31 14 4.05% 0.23% 10.00% + { 1536, 12288, 32}, // 32 8 0.52% 0.23% 9.09% + { 1664, 20480, 32}, // 33 12 2.80% 0.23% 8.33% + { 2048, 4096, 32}, // 34 2 1.54% 0.23% 23.08% + { 2432, 12288, 26}, // 35 5 1.55% 0.23% 18.75% + { 3072, 12288, 21}, // 36 4 0.52% 0.23% 26.32% + { 3584, 28672, 18}, // 37 8 0.22% 0.23% 16.67% + { 4096, 4096, 16}, // 38 1 1.54% 0.03% 14.29% + { 4224, 69632, 15}, // 39 16 3.03% 0.43% 3.12% + { 4736, 28672, 13}, // 40 6 1.11% 0.43% 12.12% + { 5376, 16384, 12}, // 41 3 1.95% 0.43% 13.51% + { 6144, 12288, 10}, // 42 2 0.52% 0.43% 14.29% + { 7168, 28672, 9}, // 43 4 0.22% 0.42% 16.67% + { 8192, 8192, 8}, // 44 1 0.78% 0.03% 14.29% +}; +#else +#error "Unsupported TCMALLOC_PAGE_SHIFT value!" +#endif +#endif +// clang-format on + +static_assert(sizeof(List) / sizeof(List[0]) <= kNumBaseClasses); +extern constexpr SizeClasses kLowFragSizeClasses{List, Assumptions}; + +// clang-format off +static_assert(SizeClassesAreDivisibleByPageSize( + kLowFragSizeClasses.classes, Bytes(kPageSize)), + "size classes must be divisible by page size"); +// clang-format on + +} // namespace tcmalloc_internal +} // namespace tcmalloc +GOOGLE_MALLOC_SECTION_END diff --git a/tcmalloc/size_classes_test.cc b/tcmalloc/size_classes_test.cc index 3b8f873a9..0d4f8ddd6 100644 --- a/tcmalloc/size_classes_test.cc +++ b/tcmalloc/size_classes_test.cc @@ -184,6 +184,7 @@ TEST_F(RunTimeSizeClassesTest, WastedSpan) { case SizeClassConfiguration::kLegacy: case SizeClassConfiguration::kReuse: case SizeClassConfiguration::kReuseRelaxedBelow64: + case SizeClassConfiguration::kLowFrag: // This test fails for other classes (was passing with a different span // size allocation algorithm used between cl/130150125 and cl/139955211). GTEST_SKIP(); diff --git a/tcmalloc/sizemap.cc b/tcmalloc/sizemap.cc index b2c0ca1b6..be277eae1 100644 --- a/tcmalloc/sizemap.cc +++ b/tcmalloc/sizemap.cc @@ -52,6 +52,8 @@ const SizeClasses& SizeMap::CurrentClasses() { case SizeClassConfiguration::kLegacy: // TODO(b/242710633): remove this opt out. return kLegacySizeClasses; + case SizeClassConfiguration::kLowFrag: + return kLowFragSizeClasses; } TC_BUG("unreachable"); } diff --git a/tcmalloc/sizemap.h b/tcmalloc/sizemap.h index 69101c532..41b02d480 100644 --- a/tcmalloc/sizemap.h +++ b/tcmalloc/sizemap.h @@ -44,12 +44,14 @@ extern const SizeClasses kSizeClasses; extern const SizeClasses kExperimentalPow2SizeClasses; extern const SizeClasses kLegacySizeClasses; extern const SizeClasses kReuseRelaxedBelow64SizeClasses; +extern const SizeClasses kLowFragSizeClasses; enum class SizeClassConfiguration { kPow2Only = 2, kLegacy = 4, kReuse = 6, kReuseRelaxedBelow64 = 8, + kLowFrag = 9, }; // Size-class information + mapping diff --git a/tcmalloc/static_vars.cc b/tcmalloc/static_vars.cc index baac988f8..d1e570248 100644 --- a/tcmalloc/static_vars.cc +++ b/tcmalloc/static_vars.cc @@ -162,10 +162,16 @@ size_t Static::pagemap_residence() { int ABSL_ATTRIBUTE_WEAK default_want_legacy_size_classes(); SizeClassConfiguration Static::size_class_configuration() { + return SizeClassConfiguration::kLowFrag; if (IsExperimentActive(Experiment::TEST_ONLY_TCMALLOC_POW2_SIZECLASS)) { return SizeClassConfiguration::kPow2Only; } + if (IsExperimentActive( + Experiment::TEST_ONLY_TCMALLOC_LOWFRAG_SIZECLASSES_V2)) { + return SizeClassConfiguration::kLowFrag; + } + // TODO(b/242710633): remove this opt out. if (default_want_legacy_size_classes != nullptr && default_want_legacy_size_classes() > 0 &&