MDEV-38918 Make large pages an explicit per-caller opt-in

include/my_sys.h

@@ -95,6 +95,7 @@ C_MODE_START
 #define MY_ROOT_USE_VMEM 0x20000U /* init_alloc_root: use my_virtual_mem_commit */
 /* Tree that should delete things automatically */
 #define MY_TREE_WITH_DELETE 0x40000U
+#define MY_TRY_LARGE_PAGES 0x80000U /* try to allocate large pages */
 
 #define MY_CHECK_ERROR	1U	/* Params to my_end; Check open-close */
 #define MY_GIVE_INFO	2U	/* Give time info about process*/
@@ -176,14 +177,19 @@ extern void my_free(void *ptr);
 extern void *my_memdup(PSI_memory_key key, const void *from,size_t length,myf MyFlags);
 extern char *my_strdup(PSI_memory_key key, const char *from,myf MyFlags);
 extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf MyFlags);
-extern my_bool my_use_large_pages;
+/**
+  0, or MY_TRY_LARGE_PAGES to request large pages from my_large_malloc(),
+  my_large_virtual_alloc(), or the my_virtual_mem_*() functions. Set once
+  by my_init_large_pages(), does not change thereafter.
+*/
+extern myf my_large_pages_flag;
 
 int my_init_large_pages(void);
 uchar *my_large_malloc(size_t *size, myf my_flags);
 #ifdef _WIN32
 /* On Windows, use my_virtual_mem_reserve() and my_virtual_mem_commit(). */
 #else
-char *my_large_virtual_alloc(size_t *size);
+char *my_large_virtual_alloc(size_t *size, myf my_flags);
 #endif
 void my_large_free(void *ptr, size_t size);
 void my_large_page_truncate(size_t *size);

include/my_virtual_mem.h

@@ -19,17 +19,18 @@
   (reserve, commit, decommit, release)
 */
 #include <stddef.h> /*size_t*/
+#include <my_global.h> /*myf*/
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 enum my_vmem_prot { MY_VMEM_READONLY= 0, MY_VMEM_READWRITE };
 
-char *my_virtual_mem_reserve(size_t *size);
-char *my_virtual_mem_commit(char *ptr, size_t size);
-void my_virtual_mem_decommit(char *ptr, size_t size);
-void my_virtual_mem_release(char *ptr, size_t size);
+char *my_virtual_mem_reserve(size_t *size, myf my_flags);
+char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags);
+void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags);
+void my_virtual_mem_release(char *ptr, size_t size, myf my_flags);
 void my_virtual_mem_protect(void *ptr, size_t size, enum my_vmem_prot prot);
 
 #ifdef __cplusplus

mysql-test/main/large_pages.result

@@ -1,4 +1,4 @@
-call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*");
+call mtr.add_suppression("\\[Warning\\] mariadbd: Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*");
 call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*");
 create table t1 (
 a int not null auto_increment,

mysql-test/main/large_pages.test

@@ -2,7 +2,7 @@
 
 --source include/have_innodb.inc
 
-call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*");
+call mtr.add_suppression("\\[Warning\\] mariadbd: Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*");
 call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*");
 create table t1 (
   a int not null auto_increment,

mysys/mf_keycache.c

@@ -549,7 +549,7 @@ int init_simple_key_cache(void *keycache_,
         blocks--;
       keycache->allocated_mem_size= blocks * keycache->key_cache_block_size;
       if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size,
-                                                MYF(0))))
+                                                my_large_pages_flag)))
       {
         /*
 	  Allocate memory for blocks, hash_links and hash entries;

mysys/my_alloc.c

@@ -52,7 +52,7 @@ static void *root_alloc(MEM_ROOT *root, size_t size, size_t *alloced_size,
   {
     void *ptr;
     *alloced_size= MY_ALIGN(size, my_system_page_size);
-    if ((ptr= my_virtual_mem_commit(NULL, *alloced_size)))
+    if ((ptr= my_virtual_mem_commit(NULL, *alloced_size, MYF(0))))
       update_malloc_size(*alloced_size,
                          MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC));
     return ptr;
@@ -67,7 +67,7 @@ static void root_free(MEM_ROOT *root, void *ptr, size_t size)
   {
     update_malloc_size(-(longlong) size,
                         MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC));
-    my_virtual_mem_release(ptr, size);
+    my_virtual_mem_release(ptr, size, MYF(0));
   }
   else
     my_free(ptr);

mysys/my_largepage.c

@@ -36,7 +36,7 @@ extern int memcntl(caddr_t, size_t, int, caddr_t, int, int);
 #endif /* HAVE_SOLARIS_LARGE_PAGES */
 
 
-my_bool my_use_large_pages;
+READ_ONLY_SYSVAR myf my_large_pages_flag;
 
 #ifdef _WIN32
 static size_t my_large_page_size;
@@ -187,7 +187,7 @@ static size_t my_next_large_page_size(size_t sz, int *start)
 
 int my_init_large_pages(void)
 {
-  my_use_large_pages= 1;
+  my_large_pages_flag= MY_TRY_LARGE_PAGES;
 #ifdef _WIN32
   if (!my_obtain_privilege(SE_LOCK_MEMORY_NAME))
   {
@@ -196,12 +196,19 @@ int my_init_large_pages(void)
                     " large-pages, see "
 		    "https://mariadb.com/docs/server/ha-and-performance/mariadb-memory-allocation#huge-pages"
                     , MYF(MY_WME));
-    my_use_large_pages= 0;
+    my_large_pages_flag= 0;
   }
   my_large_page_size= GetLargePageMinimum();
 #endif
 
   my_get_large_page_sizes(my_large_page_sizes);
+#ifndef _WIN32
+  if (!my_large_page_sizes[0])
+  {
+    /* No large page size is available; never attempt large pages. */
+    my_large_pages_flag= 0;
+  }
+#endif
 
 #ifdef HAVE_SOLARIS_LARGE_PAGES
   extern my_bool opt_super_large_pages;
@@ -256,7 +263,7 @@ int my_init_large_pages(void)
 */
 void my_large_page_truncate(size_t *size)
 {
-  if (my_use_large_pages)
+  if (my_large_pages_flag)
   {
     size_t large_page_size= 0;
 #ifdef _WIN32
@@ -295,13 +302,21 @@ MAP_ANON but MAP_ANONYMOUS is marked "for compatibility" */
 uchar *my_large_malloc(size_t *size, myf my_flags)
 {
   uchar *ptr= NULL;
+  /*
+    Only actually attempt large pages if the caller passed
+    MY_TRY_LARGE_PAGES (the caller is expected to only pass what it got
+    from my_large_pages_flag); otherwise always do a plain allocation of
+    the exact requested size, so *size is never rounded up to the large
+    page granularity.
+  */
+  const my_bool use_large_pages= (my_flags & MY_TRY_LARGE_PAGES) != 0;
 
 #ifdef _WIN32
   DWORD alloc_type= MEM_COMMIT | MEM_RESERVE;
   size_t orig_size= *size;
   DBUG_ENTER("my_large_malloc");
 
-  if (my_use_large_pages)
+  if (use_large_pages)
   {
     alloc_type|= MEM_LARGE_PAGES;
     /* Align block size to my_large_page_size */
@@ -312,7 +327,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
   {
     if (my_flags & MY_WME)
     {
-      if (my_use_large_pages)
+      if (use_large_pages)
       {
         my_printf_error(EE_OUTOFMEMORY,
                         "Couldn't allocate %zu bytes (MEM_LARGE_PAGES page "
@@ -325,7 +340,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
         my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size);
       }
     }
-    if (my_use_large_pages)
+    if (use_large_pages)
     {
       *size= orig_size;
       ptr= VirtualAlloc(NULL, *size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
@@ -345,7 +360,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
   while (1)
   {
     mapflag= MAP_PRIVATE | OS_MAP_ANON;
-    if (my_use_large_pages)
+    if (use_large_pages)
     {
       large_page_size= my_next_large_page_size(*size, &page_i);
       /* this might be 0, in which case we do a standard mmap */
@@ -432,16 +447,39 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
   Special large pages allocator, with possibility to commit to allocating
   more memory later.
   Every implementation returns a zero filled buffer here.
+  Initial protection of returned buffer is readwrite, if MY_TRY_LARGE_PAGES
+  is set in my_flags or on AIX(ask Marko why), but no access otherwise.
+  The caller is expected to call my_virtual_mem_commit() before using memory.
 */
-char *my_large_virtual_alloc(size_t *size)
+char *my_large_virtual_alloc(size_t *size, myf my_flags)
 {
   char *ptr;
+#ifdef _AIX
+  int flags= MAP_PRIVATE | OS_MAP_ANON;
+  int prot= PROT_READ | PROT_WRITE;
+#else
+  /*
+    Illumos important to have MAP_NORESERVE otherwise reserves all swap. On
+    innodb_buffer_pool_size_max overallocation.
+    Linux is controlled on sysctl vm.overcommit_memory.
+  */
+  int flags= MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE;
+  int prot= PROT_NONE;
+#endif
   DBUG_ENTER("my_large_virtual_alloc");
 
-  if (my_use_large_pages)
+  if (my_flags & MY_TRY_LARGE_PAGES)
   {
     size_t large_page_size;
     int page_i= 0;
+    /*
+      MY_TRY_LARGE_PAGES needs memory that is guaranteed to be usable right
+      away, whether or not a usable large page size is found below, so
+      MAP_NORESERVE does not apply here (matching the loop's own mapflag,
+      which never uses it either). On AIX, this is already the case.
+    */
+    prot= PROT_READ | PROT_WRITE;
+    flags= MAP_PRIVATE | OS_MAP_ANON;
 
     while ((large_page_size= my_next_large_page_size(*size, &page_i)) != 0)
     {
@@ -469,7 +507,7 @@ char *my_large_virtual_alloc(size_t *size)
         OS_MAP_ANON;
 
       size_t aligned_size= MY_ALIGN(*size, (size_t) large_page_size);
-      ptr= mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, mapflag, -1, 0);
+      ptr= mmap(NULL, aligned_size, prot, mapflag, -1, 0);
       if (ptr == MAP_FAILED)
       {
         ptr= NULL;
@@ -490,24 +528,9 @@ char *my_large_virtual_alloc(size_t *size)
         DBUG_RETURN(ptr);
       }
     }
-
-    my_use_large_pages= FALSE;
   }
 
-# ifdef _AIX
-  /* On IBM AIX, my_virtual_mem_commit() relies on mprotect(2) rather than
-  a subsequent mmap(2) with MAP_FIXED. */
-  ptr= mmap(NULL, *size, PROT_READ | PROT_WRITE,
-            MAP_PRIVATE | OS_MAP_ANON, -1, 0);
-# else
-/*
-  Illumos important to have MAP_NORESERVE otherwise reserves all swap. On
-  innodb_buffer_pool_size_max overallocation.
-  Linux is controlled on sysctl vm.overcommit_memory.
-*/
-  ptr= mmap(NULL, *size, PROT_NONE, MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE,
-            -1, 0);
-# endif
+  ptr= mmap(NULL, *size, prot, flags, -1, 0);
   if (ptr == MAP_FAILED)
     ptr= NULL;
 

mysys/my_virtual_mem.c

@@ -37,12 +37,14 @@
   InnoDB, the only user of this functionality), but it's the established
   terminology.
 
-  We try to respect use_large_pages setting, both on Windows and Linux
+  The caller requests large pages by passing my_large_pages_flag in
+  my_flags, consistently across the reserve/commit/decommit/release
+  calls for a given allocation.
 */
-char *my_virtual_mem_reserve(size_t *size)
+char *my_virtual_mem_reserve(size_t *size, myf my_flags)
 {
 #ifdef _WIN32
-  DWORD flags= my_use_large_pages
+  DWORD flags= (my_flags & MY_TRY_LARGE_PAGES)
     ? MEM_LARGE_PAGES | MEM_RESERVE | MEM_COMMIT
     : MEM_RESERVE;
   char *ptr= VirtualAlloc(NULL, *size, flags, PAGE_READWRITE);
@@ -55,7 +57,7 @@ char *my_virtual_mem_reserve(size_t *size)
   }
   return ptr;
 #else
-  return my_large_virtual_alloc(size);
+  return my_large_virtual_alloc(size, my_flags);
 #endif
 }
 
@@ -76,12 +78,12 @@ static my_bool is_memory_committed(char *ptr, size_t size)
 
   This is compatible with the mmap / VirtualAlloc semantics.
 */
-char *my_virtual_mem_commit(char *ptr, size_t size)
+char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags)
 {
 #ifdef _WIN32
   if (!ptr)
     return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
-  if (my_use_large_pages)
+  if (my_flags & MY_TRY_LARGE_PAGES)
   {
     DBUG_ASSERT(is_memory_committed(ptr, size));
   }
@@ -102,7 +104,7 @@ char *my_virtual_mem_commit(char *ptr, size_t size)
                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
     return p == MAP_FAILED ? NULL : p;
   }
-  if (my_use_large_pages)
+  if (my_flags & MY_TRY_LARGE_PAGES)
     /* my_large_virtual_alloc() already created a read/write mapping. */;
   else
   {
@@ -142,11 +144,11 @@ char *my_virtual_mem_commit(char *ptr, size_t size)
   return ptr;
 }
 
-void my_virtual_mem_decommit(char *ptr, size_t size)
+void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags)
 {
 #ifdef _WIN32
   DBUG_ASSERT(is_memory_committed(ptr, size));
-  if (!my_use_large_pages)
+  if (!(my_flags & MY_TRY_LARGE_PAGES))
   {
     if (!VirtualFree(ptr, size, MEM_DECOMMIT))
     {
@@ -183,8 +185,9 @@ void my_virtual_mem_decommit(char *ptr, size_t size)
   update_malloc_size(-(longlong) size, 0);
 }
 
-void my_virtual_mem_release(char *ptr, size_t size)
+void my_virtual_mem_release(char *ptr, size_t size, myf my_flags)
 {
+  (void) my_flags;
 #ifdef _WIN32
   if (!VirtualFree(ptr, 0, MEM_RELEASE))
   {