From 18f0d961cb6ede6e5e059023e43357049325b286 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 1 Sep 2026 12:27:29 +0200 Subject: [PATCH] MDEV-38918 Make large pages an explicit per-caller opt-in my_large_malloc() attempted large pages whenever --large-pages was enabled, silently rounding the size up and reporting it back via an in/out parameter. ut_malloc_dontdump() never passed that adjusted size on to its own callers (the InnoDB redo log buffer and recv_sys_t::tmp_buf), so freeing later used the original, smaller size, causing the reported "faux memory leak". Only the buffer pool and the MyISAM/Aria key caches are documented to benefit from large pages. Everything else that ended up calling my_large_malloc() only wanted its "do not dump to core" property and picked up large pages as an undocumented side effect; those buffers are also small and sequentially accessed, so they would have gained little from large pages anyway. Add MY_TRY_LARGE_PAGES: my_large_malloc() and my_large_virtual_alloc() now only attempt large pages when a caller passes this flag, instead of always trying whenever the global option is set. Only the buffer pool and the key caches pass it. The redo log buffer, tmp_buf, and row0log.cc's crypt buffers no longer request large pages at all, which removes the size-rounding bug for them without touching that code. my_large_virtual_alloc()'s fallback (no usable large page size) must also return read-write memory right away, like the Windows large-pages fallback already does, since my_virtual_mem_commit() is a no-op for MY_TRY_LARGE_PAGES. my_use_large_pages is now set once, in my_init_large_pages(), and never changed thereafter, on any platform. Also fix a broken mtr suppression regex in main.large_pages that would fail the test on Windows. --- include/my_sys.h | 10 +++- include/my_virtual_mem.h | 9 ++-- mysql-test/main/large_pages.result | 2 +- mysql-test/main/large_pages.test | 2 +- mysys/mf_keycache.c | 2 +- mysys/my_alloc.c | 4 +- mysys/my_largepage.c | 77 +++++++++++++++++---------- mysys/my_virtual_mem.c | 23 ++++---- storage/innobase/buf/buf0buf.cc | 23 ++++---- storage/innobase/handler/ha_innodb.cc | 2 +- storage/maria/ma_pagecache.c | 3 +- 11 files changed, 97 insertions(+), 60 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 43f263519beb1..0f49f2935c479 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -95,6 +95,7 @@ C_MODE_START #define MY_ROOT_USE_VMEM 0x20000U /* init_alloc_root: use my_virtual_mem_commit */ /* Tree that should delete things automatically */ #define MY_TREE_WITH_DELETE 0x40000U +#define MY_TRY_LARGE_PAGES 0x80000U /* try to allocate large pages */ #define MY_CHECK_ERROR 1U /* Params to my_end; Check open-close */ #define MY_GIVE_INFO 2U /* Give time info about process*/ @@ -176,14 +177,19 @@ extern void my_free(void *ptr); extern void *my_memdup(PSI_memory_key key, const void *from,size_t length,myf MyFlags); extern char *my_strdup(PSI_memory_key key, const char *from,myf MyFlags); extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf MyFlags); -extern my_bool my_use_large_pages; +/** + 0, or MY_TRY_LARGE_PAGES to request large pages from my_large_malloc(), + my_large_virtual_alloc(), or the my_virtual_mem_*() functions. Set once + by my_init_large_pages(), does not change thereafter. +*/ +extern myf my_large_pages_flag; int my_init_large_pages(void); uchar *my_large_malloc(size_t *size, myf my_flags); #ifdef _WIN32 /* On Windows, use my_virtual_mem_reserve() and my_virtual_mem_commit(). */ #else -char *my_large_virtual_alloc(size_t *size); +char *my_large_virtual_alloc(size_t *size, myf my_flags); #endif void my_large_free(void *ptr, size_t size); void my_large_page_truncate(size_t *size); diff --git a/include/my_virtual_mem.h b/include/my_virtual_mem.h index 689c75d5258e3..7be229c5bb7f0 100644 --- a/include/my_virtual_mem.h +++ b/include/my_virtual_mem.h @@ -19,6 +19,7 @@ (reserve, commit, decommit, release) */ #include /*size_t*/ +#include /*myf*/ #ifdef __cplusplus extern "C" { @@ -26,10 +27,10 @@ extern "C" { enum my_vmem_prot { MY_VMEM_READONLY= 0, MY_VMEM_READWRITE }; -char *my_virtual_mem_reserve(size_t *size); -char *my_virtual_mem_commit(char *ptr, size_t size); -void my_virtual_mem_decommit(char *ptr, size_t size); -void my_virtual_mem_release(char *ptr, size_t size); +char *my_virtual_mem_reserve(size_t *size, myf my_flags); +char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags); +void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags); +void my_virtual_mem_release(char *ptr, size_t size, myf my_flags); void my_virtual_mem_protect(void *ptr, size_t size, enum my_vmem_prot prot); #ifdef __cplusplus diff --git a/mysql-test/main/large_pages.result b/mysql-test/main/large_pages.result index c5e73f044a9a7..9e70be21b94d2 100644 --- a/mysql-test/main/large_pages.result +++ b/mysql-test/main/large_pages.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*"); +call mtr.add_suppression("\\[Warning\\] mariadbd: Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*"); call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*"); create table t1 ( a int not null auto_increment, diff --git a/mysql-test/main/large_pages.test b/mysql-test/main/large_pages.test index 7c0f497c6d31a..c0a2e9e94da04 100644 --- a/mysql-test/main/large_pages.test +++ b/mysql-test/main/large_pages.test @@ -2,7 +2,7 @@ --source include/have_innodb.inc -call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*"); +call mtr.add_suppression("\\[Warning\\] mariadbd: Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*"); call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*"); create table t1 ( a int not null auto_increment, diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index 5c57360d9fb76..06562f7432884 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -549,7 +549,7 @@ int init_simple_key_cache(void *keycache_, blocks--; keycache->allocated_mem_size= blocks * keycache->key_cache_block_size; if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size, - MYF(0)))) + my_large_pages_flag))) { /* Allocate memory for blocks, hash_links and hash entries; diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index b8cb910b20704..281992a230cb7 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -52,7 +52,7 @@ static void *root_alloc(MEM_ROOT *root, size_t size, size_t *alloced_size, { void *ptr; *alloced_size= MY_ALIGN(size, my_system_page_size); - if ((ptr= my_virtual_mem_commit(NULL, *alloced_size))) + if ((ptr= my_virtual_mem_commit(NULL, *alloced_size, MYF(0)))) update_malloc_size(*alloced_size, MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC)); return ptr; @@ -67,7 +67,7 @@ static void root_free(MEM_ROOT *root, void *ptr, size_t size) { update_malloc_size(-(longlong) size, MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC)); - my_virtual_mem_release(ptr, size); + my_virtual_mem_release(ptr, size, MYF(0)); } else my_free(ptr); diff --git a/mysys/my_largepage.c b/mysys/my_largepage.c index 731f41908d1d5..58bcf3a6cc878 100644 --- a/mysys/my_largepage.c +++ b/mysys/my_largepage.c @@ -36,7 +36,7 @@ extern int memcntl(caddr_t, size_t, int, caddr_t, int, int); #endif /* HAVE_SOLARIS_LARGE_PAGES */ -my_bool my_use_large_pages; +READ_ONLY_SYSVAR myf my_large_pages_flag; #ifdef _WIN32 static size_t my_large_page_size; @@ -187,7 +187,7 @@ static size_t my_next_large_page_size(size_t sz, int *start) int my_init_large_pages(void) { - my_use_large_pages= 1; + my_large_pages_flag= MY_TRY_LARGE_PAGES; #ifdef _WIN32 if (!my_obtain_privilege(SE_LOCK_MEMORY_NAME)) { @@ -196,12 +196,19 @@ int my_init_large_pages(void) " large-pages, see " "https://mariadb.com/docs/server/ha-and-performance/mariadb-memory-allocation#huge-pages" , MYF(MY_WME)); - my_use_large_pages= 0; + my_large_pages_flag= 0; } my_large_page_size= GetLargePageMinimum(); #endif my_get_large_page_sizes(my_large_page_sizes); +#ifndef _WIN32 + if (!my_large_page_sizes[0]) + { + /* No large page size is available; never attempt large pages. */ + my_large_pages_flag= 0; + } +#endif #ifdef HAVE_SOLARIS_LARGE_PAGES extern my_bool opt_super_large_pages; @@ -256,7 +263,7 @@ int my_init_large_pages(void) */ void my_large_page_truncate(size_t *size) { - if (my_use_large_pages) + if (my_large_pages_flag) { size_t large_page_size= 0; #ifdef _WIN32 @@ -295,13 +302,21 @@ MAP_ANON but MAP_ANONYMOUS is marked "for compatibility" */ uchar *my_large_malloc(size_t *size, myf my_flags) { uchar *ptr= NULL; + /* + Only actually attempt large pages if the caller passed + MY_TRY_LARGE_PAGES (the caller is expected to only pass what it got + from my_large_pages_flag); otherwise always do a plain allocation of + the exact requested size, so *size is never rounded up to the large + page granularity. + */ + const my_bool use_large_pages= (my_flags & MY_TRY_LARGE_PAGES) != 0; #ifdef _WIN32 DWORD alloc_type= MEM_COMMIT | MEM_RESERVE; size_t orig_size= *size; DBUG_ENTER("my_large_malloc"); - if (my_use_large_pages) + if (use_large_pages) { alloc_type|= MEM_LARGE_PAGES; /* Align block size to my_large_page_size */ @@ -312,7 +327,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) { if (my_flags & MY_WME) { - if (my_use_large_pages) + if (use_large_pages) { my_printf_error(EE_OUTOFMEMORY, "Couldn't allocate %zu bytes (MEM_LARGE_PAGES page " @@ -325,7 +340,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size); } } - if (my_use_large_pages) + if (use_large_pages) { *size= orig_size; ptr= VirtualAlloc(NULL, *size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); @@ -345,7 +360,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) while (1) { mapflag= MAP_PRIVATE | OS_MAP_ANON; - if (my_use_large_pages) + if (use_large_pages) { large_page_size= my_next_large_page_size(*size, &page_i); /* this might be 0, in which case we do a standard mmap */ @@ -432,16 +447,39 @@ uchar *my_large_malloc(size_t *size, myf my_flags) Special large pages allocator, with possibility to commit to allocating more memory later. Every implementation returns a zero filled buffer here. + Initial protection of returned buffer is readwrite, if MY_TRY_LARGE_PAGES + is set in my_flags or on AIX(ask Marko why), but no access otherwise. + The caller is expected to call my_virtual_mem_commit() before using memory. */ -char *my_large_virtual_alloc(size_t *size) +char *my_large_virtual_alloc(size_t *size, myf my_flags) { char *ptr; +#ifdef _AIX + int flags= MAP_PRIVATE | OS_MAP_ANON; + int prot= PROT_READ | PROT_WRITE; +#else + /* + Illumos important to have MAP_NORESERVE otherwise reserves all swap. On + innodb_buffer_pool_size_max overallocation. + Linux is controlled on sysctl vm.overcommit_memory. + */ + int flags= MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE; + int prot= PROT_NONE; +#endif DBUG_ENTER("my_large_virtual_alloc"); - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) { size_t large_page_size; int page_i= 0; + /* + MY_TRY_LARGE_PAGES needs memory that is guaranteed to be usable right + away, whether or not a usable large page size is found below, so + MAP_NORESERVE does not apply here (matching the loop's own mapflag, + which never uses it either). On AIX, this is already the case. + */ + prot= PROT_READ | PROT_WRITE; + flags= MAP_PRIVATE | OS_MAP_ANON; while ((large_page_size= my_next_large_page_size(*size, &page_i)) != 0) { @@ -469,7 +507,7 @@ char *my_large_virtual_alloc(size_t *size) OS_MAP_ANON; size_t aligned_size= MY_ALIGN(*size, (size_t) large_page_size); - ptr= mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, mapflag, -1, 0); + ptr= mmap(NULL, aligned_size, prot, mapflag, -1, 0); if (ptr == MAP_FAILED) { ptr= NULL; @@ -490,24 +528,9 @@ char *my_large_virtual_alloc(size_t *size) DBUG_RETURN(ptr); } } - - my_use_large_pages= FALSE; } -# ifdef _AIX - /* On IBM AIX, my_virtual_mem_commit() relies on mprotect(2) rather than - a subsequent mmap(2) with MAP_FIXED. */ - ptr= mmap(NULL, *size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | OS_MAP_ANON, -1, 0); -# else -/* - Illumos important to have MAP_NORESERVE otherwise reserves all swap. On - innodb_buffer_pool_size_max overallocation. - Linux is controlled on sysctl vm.overcommit_memory. -*/ - ptr= mmap(NULL, *size, PROT_NONE, MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE, - -1, 0); -# endif + ptr= mmap(NULL, *size, prot, flags, -1, 0); if (ptr == MAP_FAILED) ptr= NULL; diff --git a/mysys/my_virtual_mem.c b/mysys/my_virtual_mem.c index acd63a9b0a81c..66dc63d90a0e6 100644 --- a/mysys/my_virtual_mem.c +++ b/mysys/my_virtual_mem.c @@ -37,12 +37,14 @@ InnoDB, the only user of this functionality), but it's the established terminology. - We try to respect use_large_pages setting, both on Windows and Linux + The caller requests large pages by passing my_large_pages_flag in + my_flags, consistently across the reserve/commit/decommit/release + calls for a given allocation. */ -char *my_virtual_mem_reserve(size_t *size) +char *my_virtual_mem_reserve(size_t *size, myf my_flags) { #ifdef _WIN32 - DWORD flags= my_use_large_pages + DWORD flags= (my_flags & MY_TRY_LARGE_PAGES) ? MEM_LARGE_PAGES | MEM_RESERVE | MEM_COMMIT : MEM_RESERVE; char *ptr= VirtualAlloc(NULL, *size, flags, PAGE_READWRITE); @@ -55,7 +57,7 @@ char *my_virtual_mem_reserve(size_t *size) } return ptr; #else - return my_large_virtual_alloc(size); + return my_large_virtual_alloc(size, my_flags); #endif } @@ -76,12 +78,12 @@ static my_bool is_memory_committed(char *ptr, size_t size) This is compatible with the mmap / VirtualAlloc semantics. */ -char *my_virtual_mem_commit(char *ptr, size_t size) +char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags) { #ifdef _WIN32 if (!ptr) return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) { DBUG_ASSERT(is_memory_committed(ptr, size)); } @@ -102,7 +104,7 @@ char *my_virtual_mem_commit(char *ptr, size_t size) MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); return p == MAP_FAILED ? NULL : p; } - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) /* my_large_virtual_alloc() already created a read/write mapping. */; else { @@ -142,11 +144,11 @@ char *my_virtual_mem_commit(char *ptr, size_t size) return ptr; } -void my_virtual_mem_decommit(char *ptr, size_t size) +void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags) { #ifdef _WIN32 DBUG_ASSERT(is_memory_committed(ptr, size)); - if (!my_use_large_pages) + if (!(my_flags & MY_TRY_LARGE_PAGES)) { if (!VirtualFree(ptr, size, MEM_DECOMMIT)) { @@ -183,8 +185,9 @@ void my_virtual_mem_decommit(char *ptr, size_t size) update_malloc_size(-(longlong) size, 0); } -void my_virtual_mem_release(char *ptr, size_t size) +void my_virtual_mem_release(char *ptr, size_t size, myf my_flags) { + (void) my_flags; #ifdef _WIN32 if (!VirtualFree(ptr, 0, MEM_RELEASE)) { diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0b32a7beca954..b0db969b74245 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -848,7 +848,7 @@ class mem_pressure { m_num_fds= 0; - if (my_use_large_pages) + if (my_large_pages_flag) return false; static_assert(array_elements(m_fds) == (array_elements(m_triggers) + 1), @@ -1339,7 +1339,7 @@ bool buf_pool_t::create() noexcept retry: { NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; - memory_unaligned= my_virtual_mem_reserve(&size); + memory_unaligned= my_virtual_mem_reserve(&size, my_large_pages_flag); if (memory_unaligned); #if defined __aarch64__ || defined __riscv || defined __mips__ || defined __loongarch64 else if (size_in_bytes_max_default != 0 && @@ -1373,7 +1373,7 @@ bool buf_pool_t::create() noexcept if (size < size_in_bytes_max + alignment_waste) { - my_virtual_mem_release(memory_unaligned, size); + my_virtual_mem_release(memory_unaligned, size, my_large_pages_flag); size+= 1 + (~size_t(memory_unaligned) & (innodb_buffer_pool_extent_size - 1)); goto retry; @@ -1398,9 +1398,10 @@ bool buf_pool_t::create() noexcept PSI_MEMORY_CALL(memory_alloc)(mem_key_buf_buf_pool, actual_size, &owner); #endif #ifndef _AIX - if (!my_virtual_mem_commit(memory, actual_size)) + if (!my_virtual_mem_commit(memory, actual_size, my_large_pages_flag)) { - my_virtual_mem_release(memory_unaligned, size_unaligned); + my_virtual_mem_release(memory_unaligned, size_unaligned, + my_large_pages_flag); memory= nullptr; memory_unaligned= nullptr; goto oom; @@ -1575,8 +1576,9 @@ void buf_pool_t::close() noexcept owner= nullptr; #endif os_total_large_mem_allocated-= size; - my_virtual_mem_decommit(memory, size); - my_virtual_mem_release(memory_unaligned, size_unaligned); + my_virtual_mem_decommit(memory, size, my_large_pages_flag); + my_virtual_mem_release(memory_unaligned, size_unaligned, + my_large_pages_flag); memory= nullptr; memory_unaligned= nullptr; } @@ -1892,7 +1894,7 @@ inline void buf_pool_t::shrunk(size_t size, size_t reduced) noexcept guess before we invoke my_virtual_mem_decommit() below. */ latch.unlock(); } - my_virtual_mem_decommit(memory + size, reduced); + my_virtual_mem_decommit(memory + size, reduced, my_large_pages_flag); #ifdef UNIV_PFS_MEMORY PSI_MEMORY_CALL(memory_free)(mem_key_buf_buf_pool, reduced, owner); #endif @@ -1903,7 +1905,7 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept ut_ad(this == &buf_pool); mysql_mutex_assert_owner(&LOCK_global_system_variables); ut_ad(size <= size_in_bytes_max); - if (my_use_large_pages) + if (my_large_pages_flag) { my_error(ER_VARIABLE_IS_READONLY, MYF(0), "InnoDB", "innodb_buffer_pool_size", "large_pages=0"); @@ -1950,7 +1952,8 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept if (n_blocks_removed <= 0) { - if (!my_virtual_mem_commit(memory + old_size, size - old_size)) + if (!my_virtual_mem_commit(memory + old_size, size - old_size, + my_large_pages_flag)) { mysql_mutex_unlock(&mutex); sql_print_error("InnoDB: Cannot commit innodb_buffer_pool_size=%zum;" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e06ee34828398..fb84cb829768d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3778,7 +3778,7 @@ static int innodb_init_params() const size_t innodb_buffer_pool_size= buf_pool.size_in_bytes_requested; if (innodb_buffer_pool_size > buf_pool.size_in_bytes_max || - my_use_large_pages /* large_pages=ON fixes innodb_buffer_pool_size */) + my_large_pages_flag /* large_pages=ON fixes innodb_buffer_pool_size */) buf_pool.size_in_bytes_max= ut_calc_align(innodb_buffer_pool_size, innodb_buffer_pool_extent_size); diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index c925bfe54f1be..105cc14d0b980 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -855,7 +855,8 @@ size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem, /* Allocate memory for cache page buffers */ pagecache->mem_size= blocks * pagecache->block_size; if ((pagecache->block_mem= - my_large_malloc(&pagecache->mem_size, MYF(MY_WME)))) + my_large_malloc(&pagecache->mem_size, + MYF(MY_WME) | my_large_pages_flag))) { /* Allocate memory for blocks, hash_links and hash entries;