diff --git a/include/my_sys.h b/include/my_sys.h index 43f263519beb1..0f49f2935c479 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -95,6 +95,7 @@ C_MODE_START #define MY_ROOT_USE_VMEM 0x20000U /* init_alloc_root: use my_virtual_mem_commit */ /* Tree that should delete things automatically */ #define MY_TREE_WITH_DELETE 0x40000U +#define MY_TRY_LARGE_PAGES 0x80000U /* try to allocate large pages */ #define MY_CHECK_ERROR 1U /* Params to my_end; Check open-close */ #define MY_GIVE_INFO 2U /* Give time info about process*/ @@ -176,14 +177,19 @@ extern void my_free(void *ptr); extern void *my_memdup(PSI_memory_key key, const void *from,size_t length,myf MyFlags); extern char *my_strdup(PSI_memory_key key, const char *from,myf MyFlags); extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf MyFlags); -extern my_bool my_use_large_pages; +/** + 0, or MY_TRY_LARGE_PAGES to request large pages from my_large_malloc(), + my_large_virtual_alloc(), or the my_virtual_mem_*() functions. Set once + by my_init_large_pages(), does not change thereafter. +*/ +extern myf my_large_pages_flag; int my_init_large_pages(void); uchar *my_large_malloc(size_t *size, myf my_flags); #ifdef _WIN32 /* On Windows, use my_virtual_mem_reserve() and my_virtual_mem_commit(). */ #else -char *my_large_virtual_alloc(size_t *size); +char *my_large_virtual_alloc(size_t *size, myf my_flags); #endif void my_large_free(void *ptr, size_t size); void my_large_page_truncate(size_t *size); diff --git a/include/my_virtual_mem.h b/include/my_virtual_mem.h index 689c75d5258e3..2700cf8012c94 100644 --- a/include/my_virtual_mem.h +++ b/include/my_virtual_mem.h @@ -19,6 +19,7 @@ (reserve, commit, decommit, release) */ #include /*size_t*/ +#include /*myf*/ #ifdef __cplusplus extern "C" { @@ -26,9 +27,9 @@ extern "C" { enum my_vmem_prot { MY_VMEM_READONLY= 0, MY_VMEM_READWRITE }; -char *my_virtual_mem_reserve(size_t *size); -char *my_virtual_mem_commit(char *ptr, size_t size); -void my_virtual_mem_decommit(char *ptr, size_t size); +char *my_virtual_mem_reserve(size_t *size, myf my_flags); +char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags); +void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags); void my_virtual_mem_release(char *ptr, size_t size); void my_virtual_mem_protect(void *ptr, size_t size, enum my_vmem_prot prot); diff --git a/mysql-test/main/large_pages.result b/mysql-test/main/large_pages.result index c5e73f044a9a7..9e70be21b94d2 100644 --- a/mysql-test/main/large_pages.result +++ b/mysql-test/main/large_pages.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*"); +call mtr.add_suppression("\\[Warning\\] mariadbd: Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*"); call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*"); create table t1 ( a int not null auto_increment, diff --git a/mysql-test/main/large_pages.test b/mysql-test/main/large_pages.test index 7c0f497c6d31a..c0a2e9e94da04 100644 --- a/mysql-test/main/large_pages.test +++ b/mysql-test/main/large_pages.test @@ -2,7 +2,7 @@ --source include/have_innodb.inc -call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*"); +call mtr.add_suppression("\\[Warning\\] mariadbd: Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*"); call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*"); create table t1 ( a int not null auto_increment, diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index 5c57360d9fb76..06562f7432884 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -549,7 +549,7 @@ int init_simple_key_cache(void *keycache_, blocks--; keycache->allocated_mem_size= blocks * keycache->key_cache_block_size; if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size, - MYF(0)))) + my_large_pages_flag))) { /* Allocate memory for blocks, hash_links and hash entries; diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index b8cb910b20704..07893f155274a 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -52,7 +52,7 @@ static void *root_alloc(MEM_ROOT *root, size_t size, size_t *alloced_size, { void *ptr; *alloced_size= MY_ALIGN(size, my_system_page_size); - if ((ptr= my_virtual_mem_commit(NULL, *alloced_size))) + if ((ptr= my_virtual_mem_commit(NULL, *alloced_size, MYF(0)))) update_malloc_size(*alloced_size, MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC)); return ptr; diff --git a/mysys/my_largepage.c b/mysys/my_largepage.c index 731f41908d1d5..785f1386f0eba 100644 --- a/mysys/my_largepage.c +++ b/mysys/my_largepage.c @@ -36,7 +36,7 @@ extern int memcntl(caddr_t, size_t, int, caddr_t, int, int); #endif /* HAVE_SOLARIS_LARGE_PAGES */ -my_bool my_use_large_pages; +READ_ONLY_SYSVAR myf my_large_pages_flag; #ifdef _WIN32 static size_t my_large_page_size; @@ -187,21 +187,23 @@ static size_t my_next_large_page_size(size_t sz, int *start) int my_init_large_pages(void) { - my_use_large_pages= 1; #ifdef _WIN32 - if (!my_obtain_privilege(SE_LOCK_MEMORY_NAME)) - { + my_large_pages_flag= my_obtain_privilege(SE_LOCK_MEMORY_NAME) + ? MY_TRY_LARGE_PAGES : 0; + if (!my_large_pages_flag) my_printf_error(EE_PERM_LOCK_MEMORY, "Lock Pages in memory access rights required for use with" " large-pages, see " "https://mariadb.com/docs/server/ha-and-performance/mariadb-memory-allocation#huge-pages" , MYF(MY_WME)); - my_use_large_pages= 0; - } my_large_page_size= GetLargePageMinimum(); #endif my_get_large_page_sizes(my_large_page_sizes); +#ifndef _WIN32 + /* my_large_pages_flag is 0 unless a usable large page size exists. */ + my_large_pages_flag= my_large_page_sizes[0] ? MY_TRY_LARGE_PAGES : 0; +#endif #ifdef HAVE_SOLARIS_LARGE_PAGES extern my_bool opt_super_large_pages; @@ -256,7 +258,7 @@ int my_init_large_pages(void) */ void my_large_page_truncate(size_t *size) { - if (my_use_large_pages) + if (my_large_pages_flag) { size_t large_page_size= 0; #ifdef _WIN32 @@ -295,13 +297,21 @@ MAP_ANON but MAP_ANONYMOUS is marked "for compatibility" */ uchar *my_large_malloc(size_t *size, myf my_flags) { uchar *ptr= NULL; + /* + Only actually attempt large pages if the caller passed + MY_TRY_LARGE_PAGES (the caller is expected to only pass what it got + from my_large_pages_flag); otherwise always do a plain allocation of + the exact requested size, so *size is never rounded up to the large + page granularity. + */ + const my_bool use_large_pages= (my_flags & MY_TRY_LARGE_PAGES) != 0; #ifdef _WIN32 DWORD alloc_type= MEM_COMMIT | MEM_RESERVE; size_t orig_size= *size; DBUG_ENTER("my_large_malloc"); - if (my_use_large_pages) + if (use_large_pages) { alloc_type|= MEM_LARGE_PAGES; /* Align block size to my_large_page_size */ @@ -312,7 +322,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) { if (my_flags & MY_WME) { - if (my_use_large_pages) + if (use_large_pages) { my_printf_error(EE_OUTOFMEMORY, "Couldn't allocate %zu bytes (MEM_LARGE_PAGES page " @@ -325,7 +335,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size); } } - if (my_use_large_pages) + if (use_large_pages) { *size= orig_size; ptr= VirtualAlloc(NULL, *size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); @@ -345,7 +355,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) while (1) { mapflag= MAP_PRIVATE | OS_MAP_ANON; - if (my_use_large_pages) + if (use_large_pages) { large_page_size= my_next_large_page_size(*size, &page_i); /* this might be 0, in which case we do a standard mmap */ @@ -432,16 +442,39 @@ uchar *my_large_malloc(size_t *size, myf my_flags) Special large pages allocator, with possibility to commit to allocating more memory later. Every implementation returns a zero filled buffer here. + Initial protection of returned buffer is readwrite, if MY_TRY_LARGE_PAGES + is set in my_flags or on AIX(ask Marko why), but no access otherwise. + The caller is expected to call my_virtual_mem_commit() before using memory. */ -char *my_large_virtual_alloc(size_t *size) +char *my_large_virtual_alloc(size_t *size, myf my_flags) { char *ptr; +#ifdef _AIX + int flags= MAP_PRIVATE | OS_MAP_ANON; + int prot= PROT_READ | PROT_WRITE; +#else + /* + Illumos important to have MAP_NORESERVE otherwise reserves all swap. On + innodb_buffer_pool_size_max overallocation. + Linux is controlled on sysctl vm.overcommit_memory. + */ + int flags= MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE; + int prot= PROT_NONE; +#endif DBUG_ENTER("my_large_virtual_alloc"); - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) { size_t large_page_size; int page_i= 0; + /* + MY_TRY_LARGE_PAGES needs memory that is guaranteed to be usable right + away, whether or not a usable large page size is found below, so + MAP_NORESERVE does not apply here (matching the loop's own mapflag, + which never uses it either). On AIX, this is already the case. + */ + prot= PROT_READ | PROT_WRITE; + flags= MAP_PRIVATE | OS_MAP_ANON; while ((large_page_size= my_next_large_page_size(*size, &page_i)) != 0) { @@ -469,7 +502,7 @@ char *my_large_virtual_alloc(size_t *size) OS_MAP_ANON; size_t aligned_size= MY_ALIGN(*size, (size_t) large_page_size); - ptr= mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, mapflag, -1, 0); + ptr= mmap(NULL, aligned_size, prot, mapflag, -1, 0); if (ptr == MAP_FAILED) { ptr= NULL; @@ -490,24 +523,9 @@ char *my_large_virtual_alloc(size_t *size) DBUG_RETURN(ptr); } } - - my_use_large_pages= FALSE; } -# ifdef _AIX - /* On IBM AIX, my_virtual_mem_commit() relies on mprotect(2) rather than - a subsequent mmap(2) with MAP_FIXED. */ - ptr= mmap(NULL, *size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | OS_MAP_ANON, -1, 0); -# else -/* - Illumos important to have MAP_NORESERVE otherwise reserves all swap. On - innodb_buffer_pool_size_max overallocation. - Linux is controlled on sysctl vm.overcommit_memory. -*/ - ptr= mmap(NULL, *size, PROT_NONE, MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE, - -1, 0); -# endif + ptr= mmap(NULL, *size, prot, flags, -1, 0); if (ptr == MAP_FAILED) ptr= NULL; diff --git a/mysys/my_virtual_mem.c b/mysys/my_virtual_mem.c index acd63a9b0a81c..0c1ae5a761875 100644 --- a/mysys/my_virtual_mem.c +++ b/mysys/my_virtual_mem.c @@ -37,12 +37,14 @@ InnoDB, the only user of this functionality), but it's the established terminology. - We try to respect use_large_pages setting, both on Windows and Linux + The caller requests large pages by passing my_large_pages_flag in + my_flags, consistently across the reserve/commit/decommit/release + calls for a given allocation. */ -char *my_virtual_mem_reserve(size_t *size) +char *my_virtual_mem_reserve(size_t *size, myf my_flags) { #ifdef _WIN32 - DWORD flags= my_use_large_pages + DWORD flags= (my_flags & MY_TRY_LARGE_PAGES) ? MEM_LARGE_PAGES | MEM_RESERVE | MEM_COMMIT : MEM_RESERVE; char *ptr= VirtualAlloc(NULL, *size, flags, PAGE_READWRITE); @@ -55,7 +57,7 @@ char *my_virtual_mem_reserve(size_t *size) } return ptr; #else - return my_large_virtual_alloc(size); + return my_large_virtual_alloc(size, my_flags); #endif } @@ -76,12 +78,12 @@ static my_bool is_memory_committed(char *ptr, size_t size) This is compatible with the mmap / VirtualAlloc semantics. */ -char *my_virtual_mem_commit(char *ptr, size_t size) +char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags) { #ifdef _WIN32 if (!ptr) return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) { DBUG_ASSERT(is_memory_committed(ptr, size)); } @@ -102,7 +104,7 @@ char *my_virtual_mem_commit(char *ptr, size_t size) MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); return p == MAP_FAILED ? NULL : p; } - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) /* my_large_virtual_alloc() already created a read/write mapping. */; else { @@ -142,44 +144,50 @@ char *my_virtual_mem_commit(char *ptr, size_t size) return ptr; } -void my_virtual_mem_decommit(char *ptr, size_t size) +void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags) { #ifdef _WIN32 DBUG_ASSERT(is_memory_committed(ptr, size)); - if (!my_use_large_pages) +#endif + /* + For large pages, decommit is no-op, except accounting. + This mirrors virtual_mem_commit behavior. + */ + if (!(my_flags & MY_TRY_LARGE_PAGES)) { +#ifdef _WIN32 if (!VirtualFree(ptr, size, MEM_DECOMMIT)) { my_error(EE_BADMEMORYRELEASE, MYF(ME_ERROR_LOG_ONLY), ptr, size, GetLastError()); DBUG_ASSERT(0); } - } #else # ifdef _AIX - disclaim(ptr, size, DISCLAIM_ZEROMEM); + disclaim(ptr, size, DISCLAIM_ZEROMEM); # elif defined __linux__ || defined __osf__ - madvise(ptr, size, MADV_DONTNEED); /* OSF/1, Linux mimicing AIX disclaim() */ + madvise(ptr, size, MADV_DONTNEED); /* OSF/1, Linux mimicing AIX disclaim() */ # elif defined MADV_FREE_REUSABLE && defined MADV_FREE_REUSE - /* Mac OS X 10.9; undocumented in Apple macOS */ - madvise(ptr, size, MADV_FREE_REUSABLE); /* macOS mimicing AIX disclaim() */ + /* Mac OS X 10.9; undocumented in Apple macOS */ + madvise(ptr, size, MADV_FREE_REUSABLE); /* macOS mimicing AIX disclaim() */ # elif defined MADV_PURGE /* Illumos */ - madvise(ptr, size, MADV_PURGE); /* Illumos mimicing AIX disclaim() */ + madvise(ptr, size, MADV_PURGE); /* Illumos mimicing AIX disclaim() */ # elif defined MADV_FREE - /* FreeBSD, NetBSD, OpenBSD, Dragonfly BSD, OpenSolaris, Apple macOS */ - madvise(ptr, size, MADV_FREE); /* allow lazy zeroing out */ + /* FreeBSD, NetBSD, OpenBSD, Dragonfly BSD, OpenSolaris, Apple macOS */ + madvise(ptr, size, MADV_FREE); /* allow lazy zeroing out */ # elif defined MADV_DONTNEED # warning "It is unclear if madvise(MADV_DONTNEED) works as intended" - madvise(ptr, size, MADV_DONTNEED); + madvise(ptr, size, MADV_DONTNEED); # else # warning "Do not know how to decommit memory" # endif - if (mprotect(ptr, size, PROT_NONE)) - { - my_error(EE_BADMEMORYRELEASE, MYF(ME_ERROR_LOG_ONLY), ptr, size, errno); - DBUG_ASSERT(0); - } + if (mprotect(ptr, size, PROT_NONE)) + { + my_error(EE_BADMEMORYRELEASE, MYF(ME_ERROR_LOG_ONLY), ptr, size, errno); + DBUG_ASSERT(0); + } #endif + } update_malloc_size(-(longlong) size, 0); } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0b32a7beca954..d0e4fd121cec7 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -848,7 +848,7 @@ class mem_pressure { m_num_fds= 0; - if (my_use_large_pages) + if (my_large_pages_flag) return false; static_assert(array_elements(m_fds) == (array_elements(m_triggers) + 1), @@ -1339,7 +1339,7 @@ bool buf_pool_t::create() noexcept retry: { NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; - memory_unaligned= my_virtual_mem_reserve(&size); + memory_unaligned= my_virtual_mem_reserve(&size, my_large_pages_flag); if (memory_unaligned); #if defined __aarch64__ || defined __riscv || defined __mips__ || defined __loongarch64 else if (size_in_bytes_max_default != 0 && @@ -1398,7 +1398,7 @@ bool buf_pool_t::create() noexcept PSI_MEMORY_CALL(memory_alloc)(mem_key_buf_buf_pool, actual_size, &owner); #endif #ifndef _AIX - if (!my_virtual_mem_commit(memory, actual_size)) + if (!my_virtual_mem_commit(memory, actual_size, my_large_pages_flag)) { my_virtual_mem_release(memory_unaligned, size_unaligned); memory= nullptr; @@ -1575,7 +1575,7 @@ void buf_pool_t::close() noexcept owner= nullptr; #endif os_total_large_mem_allocated-= size; - my_virtual_mem_decommit(memory, size); + my_virtual_mem_decommit(memory, size, my_large_pages_flag); my_virtual_mem_release(memory_unaligned, size_unaligned); memory= nullptr; memory_unaligned= nullptr; @@ -1892,7 +1892,7 @@ inline void buf_pool_t::shrunk(size_t size, size_t reduced) noexcept guess before we invoke my_virtual_mem_decommit() below. */ latch.unlock(); } - my_virtual_mem_decommit(memory + size, reduced); + my_virtual_mem_decommit(memory + size, reduced, my_large_pages_flag); #ifdef UNIV_PFS_MEMORY PSI_MEMORY_CALL(memory_free)(mem_key_buf_buf_pool, reduced, owner); #endif @@ -1903,7 +1903,7 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept ut_ad(this == &buf_pool); mysql_mutex_assert_owner(&LOCK_global_system_variables); ut_ad(size <= size_in_bytes_max); - if (my_use_large_pages) + if (my_large_pages_flag) { my_error(ER_VARIABLE_IS_READONLY, MYF(0), "InnoDB", "innodb_buffer_pool_size", "large_pages=0"); @@ -1950,7 +1950,8 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept if (n_blocks_removed <= 0) { - if (!my_virtual_mem_commit(memory + old_size, size - old_size)) + if (!my_virtual_mem_commit(memory + old_size, size - old_size, + my_large_pages_flag)) { mysql_mutex_unlock(&mutex); sql_print_error("InnoDB: Cannot commit innodb_buffer_pool_size=%zum;" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e06ee34828398..fb84cb829768d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3778,7 +3778,7 @@ static int innodb_init_params() const size_t innodb_buffer_pool_size= buf_pool.size_in_bytes_requested; if (innodb_buffer_pool_size > buf_pool.size_in_bytes_max || - my_use_large_pages /* large_pages=ON fixes innodb_buffer_pool_size */) + my_large_pages_flag /* large_pages=ON fixes innodb_buffer_pool_size */) buf_pool.size_in_bytes_max= ut_calc_align(innodb_buffer_pool_size, innodb_buffer_pool_extent_size); diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index c925bfe54f1be..105cc14d0b980 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -855,7 +855,8 @@ size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem, /* Allocate memory for cache page buffers */ pagecache->mem_size= blocks * pagecache->block_size; if ((pagecache->block_mem= - my_large_malloc(&pagecache->mem_size, MYF(MY_WME)))) + my_large_malloc(&pagecache->mem_size, + MYF(MY_WME) | my_large_pages_flag))) { /* Allocate memory for blocks, hash_links and hash entries;