wtf/TCSystemAlloc.cpp

   1 // Copyright (c) 2005, 2007, Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 //     * Redistributions of source code must retain the above copyright
   9 // notice, this list of conditions and the following disclaimer.
  10 //     * Redistributions in binary form must reproduce the above
  11 // copyright notice, this list of conditions and the following disclaimer
  12 // in the documentation and/or other materials provided with the
  13 // distribution.
  14 //     * Neither the name of Google Inc. nor the names of its
  15 // contributors may be used to endorse or promote products derived from
  16 // this software without specific prior written permission.
  17 //
  18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 // ---
  31 // Author: Sanjay Ghemawat
  32
  33 #include "config.h"
  34 #include "TCSystemAlloc.h"
  35
  36 #include <algorithm>
  37 #include <fcntl.h>
  38 #include "Assertions.h"
  39 #include "TCSpinLock.h"
  40 #include "UnusedParam.h"
  41 #include "VMTags.h"
  42
  43 #if HAVE(STDINT_H)
  44 #include <stdint.h>
  45 #elif HAVE(INTTYPES_H)
  46 #include <inttypes.h>
  47 #else
  48 #include <sys/types.h>
  49 #endif
  50
  51 #if OS(WINDOWS)
  52 #include "windows.h"
  53 #else
  54 #include <errno.h>
  55 #include <unistd.h>
  56 #include <sys/mman.h>
  57 #endif
  58
  59 #ifndef MAP_ANONYMOUS
  60 #define MAP_ANONYMOUS MAP_ANON
  61 #endif
  62
  63 using namespace std;
  64
  65 // Structure for discovering alignment
  66 union MemoryAligner {
  67   void*  p;
  68   double d;
  69   size_t s;
  70 };
  71
  72 static SpinLock spinlock = SPINLOCK_INITIALIZER;
  73
  74 // Page size is initialized on demand
  75 static size_t pagesize = 0;
  76
  77 // Configuration parameters.
  78 //
  79 // if use_devmem is true, either use_sbrk or use_mmap must also be true.
  80 // For 2.2 kernels, it looks like the sbrk address space (500MBish) and
  81 // the mmap address space (1300MBish) are disjoint, so we need both allocators
  82 // to get as much virtual memory as possible.
  83 #ifndef WTF_CHANGES
  84 static bool use_devmem = false;
  85 #endif
  86
  87 #if HAVE(SBRK)
  88 static bool use_sbrk = false;
  89 #endif
  90
  91 #if HAVE(MMAP)
  92 static bool use_mmap = true;
  93 #endif
  94
  95 #if HAVE(VIRTUALALLOC)
  96 static bool use_VirtualAlloc = true;
  97 #endif
  98
  99 // Flags to keep us from retrying allocators that failed.
 100 static bool devmem_failure = false;
 101 static bool sbrk_failure = false;
 102 static bool mmap_failure = false;
 103 static bool VirtualAlloc_failure = false;
 104
 105 #ifndef WTF_CHANGES
 106 DEFINE_int32(malloc_devmem_start, 0,
 107              "Physical memory starting location in MB for /dev/mem allocation."
 108              "  Setting this to 0 disables /dev/mem allocation");
 109 DEFINE_int32(malloc_devmem_limit, 0,
 110              "Physical memory limit location in MB for /dev/mem allocation."
 111              "  Setting this to 0 means no limit.");
 112 #else
 113 static const int32_t FLAGS_malloc_devmem_start = 0;
 114 static const int32_t FLAGS_malloc_devmem_limit = 0;
 115 #endif
 116
 117 #if HAVE(SBRK)
 118
 119 static void* TrySbrk(size_t size, size_t *actual_size, size_t alignment) {
 120   size = ((size + alignment - 1) / alignment) * alignment;
 121
 122   // could theoretically return the "extra" bytes here, but this
 123   // is simple and correct.
 124   if (actual_size)
 125     *actual_size = size;
 126
 127   void* result = sbrk(size);
 128   if (result == reinterpret_cast<void*>(-1)) {
 129     sbrk_failure = true;
 130     return NULL;
 131   }
 132
 133   // Is it aligned?
 134   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 135   if ((ptr & (alignment-1)) == 0)  return result;
 136
 137   // Try to get more memory for alignment
 138   size_t extra = alignment - (ptr & (alignment-1));
 139   void* r2 = sbrk(extra);
 140   if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
 141     // Contiguous with previous result
 142     return reinterpret_cast<void*>(ptr + extra);
 143   }
 144
 145   // Give up and ask for "size + alignment - 1" bytes so
 146   // that we can find an aligned region within it.
 147   result = sbrk(size + alignment - 1);
 148   if (result == reinterpret_cast<void*>(-1)) {
 149     sbrk_failure = true;
 150     return NULL;
 151   }
 152   ptr = reinterpret_cast<uintptr_t>(result);
 153   if ((ptr & (alignment-1)) != 0) {
 154     ptr += alignment - (ptr & (alignment-1));
 155   }
 156   return reinterpret_cast<void*>(ptr);
 157 }
 158
 159 #endif /* HAVE(SBRK) */
 160
 161 #if HAVE(MMAP)
 162
 163 static void* TryMmap(size_t size, size_t *actual_size, size_t alignment) {
 164   // Enforce page alignment
 165   if (pagesize == 0) pagesize = getpagesize();
 166   if (alignment < pagesize) alignment = pagesize;
 167   size = ((size + alignment - 1) / alignment) * alignment;
 168
 169   // could theoretically return the "extra" bytes here, but this
 170   // is simple and correct.
 171   if (actual_size)
 172     *actual_size = size;
 173
 174   // Ask for extra memory if alignment > pagesize
 175   size_t extra = 0;
 176   if (alignment > pagesize) {
 177     extra = alignment - pagesize;
 178   }
 179   void* result = mmap(NULL, size + extra,
 180                       PROT_READ | PROT_WRITE,
 181                       MAP_PRIVATE|MAP_ANONYMOUS,
 182                       VM_TAG_FOR_TCMALLOC_MEMORY, 0);
 183   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
 184     mmap_failure = true;
 185     return NULL;
 186   }
 187
 188   // Adjust the return memory so it is aligned
 189   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 190   size_t adjust = 0;
 191   if ((ptr & (alignment - 1)) != 0) {
 192     adjust = alignment - (ptr & (alignment - 1));
 193   }
 194
 195   // Return the unused memory to the system
 196   if (adjust > 0) {
 197     munmap(reinterpret_cast<void*>(ptr), adjust);
 198   }
 199   if (adjust < extra) {
 200     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
 201   }
 202
 203   ptr += adjust;
 204   return reinterpret_cast<void*>(ptr);
 205 }
 206
 207 #endif /* HAVE(MMAP) */
 208
 209 #if HAVE(VIRTUALALLOC)
 210
 211 static void* TryVirtualAlloc(size_t size, size_t *actual_size, size_t alignment) {
 212   // Enforce page alignment
 213   if (pagesize == 0) {
 214     SYSTEM_INFO system_info;
 215     GetSystemInfo(&system_info);
 216     pagesize = system_info.dwPageSize;
 217   }
 218
 219   if (alignment < pagesize) alignment = pagesize;
 220   size = ((size + alignment - 1) / alignment) * alignment;
 221
 222   // could theoretically return the "extra" bytes here, but this
 223   // is simple and correct.
 224   if (actual_size)
 225     *actual_size = size;
 226
 227   // Ask for extra memory if alignment > pagesize
 228   size_t extra = 0;
 229   if (alignment > pagesize) {
 230     extra = alignment - pagesize;
 231   }
 232   void* result = VirtualAlloc(NULL, size + extra,
 233                               MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN,
 234                               PAGE_READWRITE);
 235
 236   if (result == NULL) {
 237     VirtualAlloc_failure = true;
 238     return NULL;
 239   }
 240
 241   // Adjust the return memory so it is aligned
 242   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 243   size_t adjust = 0;
 244   if ((ptr & (alignment - 1)) != 0) {
 245     adjust = alignment - (ptr & (alignment - 1));
 246   }
 247
 248   // Return the unused memory to the system - we'd like to release but the best we can do
 249   // is decommit, since Windows only lets you free the whole allocation.
 250   if (adjust > 0) {
 251     VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT);
 252   }
 253   if (adjust < extra) {
 254     VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT);
 255   }
 256
 257   ptr += adjust;
 258   return reinterpret_cast<void*>(ptr);
 259 }
 260
 261 #endif /* HAVE(MMAP) */
 262
 263 #ifndef WTF_CHANGES
 264 static void* TryDevMem(size_t size, size_t *actual_size, size_t alignment) {
 265   static bool initialized = false;
 266   static off_t physmem_base;  // next physical memory address to allocate
 267   static off_t physmem_limit; // maximum physical address allowed
 268   static int physmem_fd;      // file descriptor for /dev/mem
 269
 270   // Check if we should use /dev/mem allocation.  Note that it may take
 271   // a while to get this flag initialized, so meanwhile we fall back to
 272   // the next allocator.  (It looks like 7MB gets allocated before
 273   // this flag gets initialized -khr.)
 274   if (FLAGS_malloc_devmem_start == 0) {
 275     // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
 276     // try us again next time.
 277     return NULL;
 278   }
 279
 280   if (!initialized) {
 281     physmem_fd = open("/dev/mem", O_RDWR);
 282     if (physmem_fd < 0) {
 283       devmem_failure = true;
 284       return NULL;
 285     }
 286     physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
 287     physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
 288     initialized = true;
 289   }
 290
 291   // Enforce page alignment
 292   if (pagesize == 0) pagesize = getpagesize();
 293   if (alignment < pagesize) alignment = pagesize;
 294   size = ((size + alignment - 1) / alignment) * alignment;
 295
 296   // could theoretically return the "extra" bytes here, but this
 297   // is simple and correct.
 298   if (actual_size)
 299     *actual_size = size;
 300
 301   // Ask for extra memory if alignment > pagesize
 302   size_t extra = 0;
 303   if (alignment > pagesize) {
 304     extra = alignment - pagesize;
 305   }
 306
 307   // check to see if we have any memory left
 308   if (physmem_limit != 0 && physmem_base + size + extra > physmem_limit) {
 309     devmem_failure = true;
 310     return NULL;
 311   }
 312   void *result = mmap(0, size + extra, PROT_READ | PROT_WRITE,
 313                       MAP_SHARED, physmem_fd, physmem_base);
 314   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
 315     devmem_failure = true;
 316     return NULL;
 317   }
 318   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 319
 320   // Adjust the return memory so it is aligned
 321   size_t adjust = 0;
 322   if ((ptr & (alignment - 1)) != 0) {
 323     adjust = alignment - (ptr & (alignment - 1));
 324   }
 325
 326   // Return the unused virtual memory to the system
 327   if (adjust > 0) {
 328     munmap(reinterpret_cast<void*>(ptr), adjust);
 329   }
 330   if (adjust < extra) {
 331     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
 332   }
 333
 334   ptr += adjust;
 335   physmem_base += adjust + size;
 336
 337   return reinterpret_cast<void*>(ptr);
 338 }
 339 #endif
 340
 341 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) {
 342   // Discard requests that overflow
 343   if (size + alignment < size) return NULL;
 344
 345   SpinLockHolder lock_holder(&spinlock);
 346
 347   // Enforce minimum alignment
 348   if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
 349
 350   // Try twice, once avoiding allocators that failed before, and once
 351   // more trying all allocators even if they failed before.
 352   for (int i = 0; i < 2; i++) {
 353
 354 #ifndef WTF_CHANGES
 355     if (use_devmem && !devmem_failure) {
 356       void* result = TryDevMem(size, actual_size, alignment);
 357       if (result != NULL) return result;
 358     }
 359 #endif
 360
 361 #if HAVE(SBRK)
 362     if (use_sbrk && !sbrk_failure) {
 363       void* result = TrySbrk(size, actual_size, alignment);
 364       if (result != NULL) return result;
 365     }
 366 #endif
 367
 368 #if HAVE(MMAP)
 369     if (use_mmap && !mmap_failure) {
 370       void* result = TryMmap(size, actual_size, alignment);
 371       if (result != NULL) return result;
 372     }
 373 #endif
 374
 375 #if HAVE(VIRTUALALLOC)
 376     if (use_VirtualAlloc && !VirtualAlloc_failure) {
 377       void* result = TryVirtualAlloc(size, actual_size, alignment);
 378       if (result != NULL) return result;
 379     }
 380 #endif
 381
 382     // nothing worked - reset failure flags and try again
 383     devmem_failure = false;
 384     sbrk_failure = false;
 385     mmap_failure = false;
 386     VirtualAlloc_failure = false;
 387   }
 388   return NULL;
 389 }
 390
 391 #if HAVE(MADV_FREE_REUSE)
 392
 393 void TCMalloc_SystemRelease(void* start, size_t length)
 394 {
 395     while (madvise(start, length, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) { }
 396 }
 397
 398 #elif HAVE(MADV_FREE) || HAVE(MADV_DONTNEED)
 399
 400 void TCMalloc_SystemRelease(void* start, size_t length)
 401 {
 402     // MADV_FREE clears the modified bit on pages, which allows
 403     // them to be discarded immediately.
 404 #if HAVE(MADV_FREE)
 405     const int advice = MADV_FREE;
 406 #else
 407     const int advice = MADV_DONTNEED;
 408 #endif
 409   if (FLAGS_malloc_devmem_start) {
 410     // It's not safe to use MADV_DONTNEED if we've been mapping
 411     // /dev/mem for heap memory
 412     return;
 413   }
 414   if (pagesize == 0) pagesize = getpagesize();
 415   const size_t pagemask = pagesize - 1;
 416
 417   size_t new_start = reinterpret_cast<size_t>(start);
 418   size_t end = new_start + length;
 419   size_t new_end = end;
 420
 421   // Round up the starting address and round down the ending address
 422   // to be page aligned:
 423   new_start = (new_start + pagesize - 1) & ~pagemask;
 424   new_end = new_end & ~pagemask;
 425
 426   ASSERT((new_start & pagemask) == 0);
 427   ASSERT((new_end & pagemask) == 0);
 428   ASSERT(new_start >= reinterpret_cast<size_t>(start));
 429   ASSERT(new_end <= end);
 430
 431   if (new_end > new_start) {
 432     // Note -- ignoring most return codes, because if this fails it
 433     // doesn't matter...
 434     while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
 435                    advice) == -1 &&
 436            errno == EAGAIN) {
 437       // NOP
 438     }
 439   }
 440 }
 441
 442 #elif HAVE(MMAP)
 443
 444 void TCMalloc_SystemRelease(void* start, size_t length)
 445 {
 446   void* newAddress = mmap(start, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
 447   // If the mmap failed then that's ok, we just won't return the memory to the system.
 448   ASSERT_UNUSED(newAddress, newAddress == start || newAddress == reinterpret_cast<void*>(MAP_FAILED));
 449 }
 450
 451 #elif HAVE(VIRTUALALLOC)
 452
 453 void TCMalloc_SystemRelease(void* start, size_t length)
 454 {
 455     if (VirtualFree(start, length, MEM_DECOMMIT))
 456         return;
 457
 458     // The decommit may fail if the memory region consists of allocations
 459     // from more than one call to VirtualAlloc.  In this case, fall back to
 460     // using VirtualQuery to retrieve the allocation boundaries and decommit
 461     // them each individually.
 462
 463     char* ptr = static_cast<char*>(start);
 464     char* end = ptr + length;
 465     MEMORY_BASIC_INFORMATION info;
 466     while (ptr < end) {
 467         size_t resultSize = VirtualQuery(ptr, &info, sizeof(info));
 468         ASSERT_UNUSED(resultSize, resultSize == sizeof(info));
 469
 470         size_t decommitSize = min<size_t>(info.RegionSize, end - ptr);
 471         BOOL success = VirtualFree(ptr, decommitSize, MEM_DECOMMIT);
 472         ASSERT_UNUSED(success, success);
 473         ptr += decommitSize;
 474     }
 475 }
 476
 477 #else
 478
 479 // Platforms that don't support returning memory use an empty inline version of TCMalloc_SystemRelease
 480 // declared in TCSystemAlloc.h
 481
 482 #endif
 483
 484 #if HAVE(MADV_FREE_REUSE)
 485
 486 void TCMalloc_SystemCommit(void* start, size_t length)
 487 {
 488     while (madvise(start, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { }
 489 }
 490
 491 #elif HAVE(VIRTUALALLOC)
 492
 493 void TCMalloc_SystemCommit(void* start, size_t length)
 494 {
 495     if (VirtualAlloc(start, length, MEM_COMMIT, PAGE_READWRITE) == start)
 496         return;
 497
 498     // The commit may fail if the memory region consists of allocations
 499     // from more than one call to VirtualAlloc.  In this case, fall back to
 500     // using VirtualQuery to retrieve the allocation boundaries and commit them
 501     // each individually.
 502
 503     char* ptr = static_cast<char*>(start);
 504     char* end = ptr + length;
 505     MEMORY_BASIC_INFORMATION info;
 506     while (ptr < end) {
 507         size_t resultSize = VirtualQuery(ptr, &info, sizeof(info));
 508         ASSERT_UNUSED(resultSize, resultSize == sizeof(info));
 509
 510         size_t commitSize = min<size_t>(info.RegionSize, end - ptr);
 511         void* newAddress = VirtualAlloc(ptr, commitSize, MEM_COMMIT, PAGE_READWRITE);
 512         ASSERT_UNUSED(newAddress, newAddress == ptr);
 513         ptr += commitSize;
 514     }
 515 }
 516
 517 #else
 518
 519 // Platforms that don't need to explicitly commit memory use an empty inline version of TCMalloc_SystemCommit
 520 // declared in TCSystemAlloc.h
 521
 522 #endif