wtf/TCSystemAlloc.cpp

   1 // Copyright (c) 2005, 2007, Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 //     * Redistributions of source code must retain the above copyright
   9 // notice, this list of conditions and the following disclaimer.
  10 //     * Redistributions in binary form must reproduce the above
  11 // copyright notice, this list of conditions and the following disclaimer
  12 // in the documentation and/or other materials provided with the
  13 // distribution.
  14 //     * Neither the name of Google Inc. nor the names of its
  15 // contributors may be used to endorse or promote products derived from
  16 // this software without specific prior written permission.
  17 //
  18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 // ---
  31 // Author: Sanjay Ghemawat
  32
  33 #include "config.h"
  34 #include "TCSystemAlloc.h"
  35
  36 #include <algorithm>
  37 #include <fcntl.h>
  38 #include "Assertions.h"
  39 #include "TCSpinLock.h"
  40 #include "UnusedParam.h"
  41
  42 #if HAVE(STDINT_H)
  43 #include <stdint.h>
  44 #elif HAVE(INTTYPES_H)
  45 #include <inttypes.h>
  46 #else
  47 #include <sys/types.h>
  48 #endif
  49
  50 #if PLATFORM(WIN_OS)
  51 #include "windows.h"
  52 #else
  53 #include <errno.h>
  54 #include <unistd.h>
  55 #include <sys/mman.h>
  56 #endif
  57
  58 #ifndef MAP_ANONYMOUS
  59 #define MAP_ANONYMOUS MAP_ANON
  60 #endif
  61
  62 using namespace std;
  63
  64 // Structure for discovering alignment
  65 union MemoryAligner {
  66   void*  p;
  67   double d;
  68   size_t s;
  69 };
  70
  71 static SpinLock spinlock = SPINLOCK_INITIALIZER;
  72
  73 // Page size is initialized on demand
  74 static size_t pagesize = 0;
  75
  76 // Configuration parameters.
  77 //
  78 // if use_devmem is true, either use_sbrk or use_mmap must also be true.
  79 // For 2.2 kernels, it looks like the sbrk address space (500MBish) and
  80 // the mmap address space (1300MBish) are disjoint, so we need both allocators
  81 // to get as much virtual memory as possible.
  82 #ifndef WTF_CHANGES
  83 static bool use_devmem = false;
  84 #endif
  85
  86 #if HAVE(SBRK)
  87 static bool use_sbrk = false;
  88 #endif
  89
  90 #if HAVE(MMAP)
  91 static bool use_mmap = true;
  92 #endif
  93
  94 #if HAVE(VIRTUALALLOC)
  95 static bool use_VirtualAlloc = true;
  96 #endif
  97
  98 // Flags to keep us from retrying allocators that failed.
  99 static bool devmem_failure = false;
 100 static bool sbrk_failure = false;
 101 static bool mmap_failure = false;
 102 static bool VirtualAlloc_failure = false;
 103
 104 #ifndef WTF_CHANGES
 105 DEFINE_int32(malloc_devmem_start, 0,
 106              "Physical memory starting location in MB for /dev/mem allocation."
 107              "  Setting this to 0 disables /dev/mem allocation");
 108 DEFINE_int32(malloc_devmem_limit, 0,
 109              "Physical memory limit location in MB for /dev/mem allocation."
 110              "  Setting this to 0 means no limit.");
 111 #else
 112 static const int32_t FLAGS_malloc_devmem_start = 0;
 113 static const int32_t FLAGS_malloc_devmem_limit = 0;
 114 #endif
 115
 116 #if HAVE(SBRK)
 117
 118 static void* TrySbrk(size_t size, size_t *actual_size, size_t alignment) {
 119   size = ((size + alignment - 1) / alignment) * alignment;
 120
 121   // could theoretically return the "extra" bytes here, but this
 122   // is simple and correct.
 123   if (actual_size)
 124     *actual_size = size;
 125
 126   void* result = sbrk(size);
 127   if (result == reinterpret_cast<void*>(-1)) {
 128     sbrk_failure = true;
 129     return NULL;
 130   }
 131
 132   // Is it aligned?
 133   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 134   if ((ptr & (alignment-1)) == 0)  return result;
 135
 136   // Try to get more memory for alignment
 137   size_t extra = alignment - (ptr & (alignment-1));
 138   void* r2 = sbrk(extra);
 139   if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
 140     // Contiguous with previous result
 141     return reinterpret_cast<void*>(ptr + extra);
 142   }
 143
 144   // Give up and ask for "size + alignment - 1" bytes so
 145   // that we can find an aligned region within it.
 146   result = sbrk(size + alignment - 1);
 147   if (result == reinterpret_cast<void*>(-1)) {
 148     sbrk_failure = true;
 149     return NULL;
 150   }
 151   ptr = reinterpret_cast<uintptr_t>(result);
 152   if ((ptr & (alignment-1)) != 0) {
 153     ptr += alignment - (ptr & (alignment-1));
 154   }
 155   return reinterpret_cast<void*>(ptr);
 156 }
 157
 158 #endif /* HAVE(SBRK) */
 159
 160 #if HAVE(MMAP)
 161
 162 static void* TryMmap(size_t size, size_t *actual_size, size_t alignment) {
 163   // Enforce page alignment
 164   if (pagesize == 0) pagesize = getpagesize();
 165   if (alignment < pagesize) alignment = pagesize;
 166   size = ((size + alignment - 1) / alignment) * alignment;
 167
 168   // could theoretically return the "extra" bytes here, but this
 169   // is simple and correct.
 170   if (actual_size)
 171     *actual_size = size;
 172
 173   // Ask for extra memory if alignment > pagesize
 174   size_t extra = 0;
 175   if (alignment > pagesize) {
 176     extra = alignment - pagesize;
 177   }
 178   void* result = mmap(NULL, size + extra,
 179                       PROT_READ | PROT_WRITE,
 180                       MAP_PRIVATE|MAP_ANONYMOUS,
 181                       -1, 0);
 182   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
 183     mmap_failure = true;
 184     return NULL;
 185   }
 186
 187   // Adjust the return memory so it is aligned
 188   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 189   size_t adjust = 0;
 190   if ((ptr & (alignment - 1)) != 0) {
 191     adjust = alignment - (ptr & (alignment - 1));
 192   }
 193
 194   // Return the unused memory to the system
 195   if (adjust > 0) {
 196     munmap(reinterpret_cast<void*>(ptr), adjust);
 197   }
 198   if (adjust < extra) {
 199     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
 200   }
 201
 202   ptr += adjust;
 203   return reinterpret_cast<void*>(ptr);
 204 }
 205
 206 #endif /* HAVE(MMAP) */
 207
 208 #if HAVE(VIRTUALALLOC)
 209
 210 static void* TryVirtualAlloc(size_t size, size_t *actual_size, size_t alignment) {
 211   // Enforce page alignment
 212   if (pagesize == 0) {
 213     SYSTEM_INFO system_info;
 214     GetSystemInfo(&system_info);
 215     pagesize = system_info.dwPageSize;
 216   }
 217
 218   if (alignment < pagesize) alignment = pagesize;
 219   size = ((size + alignment - 1) / alignment) * alignment;
 220
 221   // could theoretically return the "extra" bytes here, but this
 222   // is simple and correct.
 223   if (actual_size)
 224     *actual_size = size;
 225
 226   // Ask for extra memory if alignment > pagesize
 227   size_t extra = 0;
 228   if (alignment > pagesize) {
 229     extra = alignment - pagesize;
 230   }
 231   void* result = VirtualAlloc(NULL, size + extra,
 232                               MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN,
 233                               PAGE_READWRITE);
 234
 235   if (result == NULL) {
 236     VirtualAlloc_failure = true;
 237     return NULL;
 238   }
 239
 240   // Adjust the return memory so it is aligned
 241   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 242   size_t adjust = 0;
 243   if ((ptr & (alignment - 1)) != 0) {
 244     adjust = alignment - (ptr & (alignment - 1));
 245   }
 246
 247   // Return the unused memory to the system - we'd like to release but the best we can do
 248   // is decommit, since Windows only lets you free the whole allocation.
 249   if (adjust > 0) {
 250     VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT);
 251   }
 252   if (adjust < extra) {
 253     VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT);
 254   }
 255
 256   ptr += adjust;
 257   return reinterpret_cast<void*>(ptr);
 258 }
 259
 260 #endif /* HAVE(MMAP) */
 261
 262 #ifndef WTF_CHANGES
 263 static void* TryDevMem(size_t size, size_t *actual_size, size_t alignment) {
 264   static bool initialized = false;
 265   static off_t physmem_base;  // next physical memory address to allocate
 266   static off_t physmem_limit; // maximum physical address allowed
 267   static int physmem_fd;      // file descriptor for /dev/mem
 268
 269   // Check if we should use /dev/mem allocation.  Note that it may take
 270   // a while to get this flag initialized, so meanwhile we fall back to
 271   // the next allocator.  (It looks like 7MB gets allocated before
 272   // this flag gets initialized -khr.)
 273   if (FLAGS_malloc_devmem_start == 0) {
 274     // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
 275     // try us again next time.
 276     return NULL;
 277   }
 278
 279   if (!initialized) {
 280     physmem_fd = open("/dev/mem", O_RDWR);
 281     if (physmem_fd < 0) {
 282       devmem_failure = true;
 283       return NULL;
 284     }
 285     physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
 286     physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
 287     initialized = true;
 288   }
 289
 290   // Enforce page alignment
 291   if (pagesize == 0) pagesize = getpagesize();
 292   if (alignment < pagesize) alignment = pagesize;
 293   size = ((size + alignment - 1) / alignment) * alignment;
 294
 295   // could theoretically return the "extra" bytes here, but this
 296   // is simple and correct.
 297   if (actual_size)
 298     *actual_size = size;
 299
 300   // Ask for extra memory if alignment > pagesize
 301   size_t extra = 0;
 302   if (alignment > pagesize) {
 303     extra = alignment - pagesize;
 304   }
 305
 306   // check to see if we have any memory left
 307   if (physmem_limit != 0 && physmem_base + size + extra > physmem_limit) {
 308     devmem_failure = true;
 309     return NULL;
 310   }
 311   void *result = mmap(0, size + extra, PROT_READ | PROT_WRITE,
 312                       MAP_SHARED, physmem_fd, physmem_base);
 313   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
 314     devmem_failure = true;
 315     return NULL;
 316   }
 317   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 318
 319   // Adjust the return memory so it is aligned
 320   size_t adjust = 0;
 321   if ((ptr & (alignment - 1)) != 0) {
 322     adjust = alignment - (ptr & (alignment - 1));
 323   }
 324
 325   // Return the unused virtual memory to the system
 326   if (adjust > 0) {
 327     munmap(reinterpret_cast<void*>(ptr), adjust);
 328   }
 329   if (adjust < extra) {
 330     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
 331   }
 332
 333   ptr += adjust;
 334   physmem_base += adjust + size;
 335
 336   return reinterpret_cast<void*>(ptr);
 337 }
 338 #endif
 339
 340 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) {
 341   // Discard requests that overflow
 342   if (size + alignment < size) return NULL;
 343
 344   SpinLockHolder lock_holder(&spinlock);
 345
 346   // Enforce minimum alignment
 347   if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
 348
 349   // Try twice, once avoiding allocators that failed before, and once
 350   // more trying all allocators even if they failed before.
 351   for (int i = 0; i < 2; i++) {
 352
 353 #ifndef WTF_CHANGES
 354     if (use_devmem && !devmem_failure) {
 355       void* result = TryDevMem(size, actual_size, alignment);
 356       if (result != NULL) return result;
 357     }
 358 #endif
 359
 360 #if HAVE(SBRK)
 361     if (use_sbrk && !sbrk_failure) {
 362       void* result = TrySbrk(size, actual_size, alignment);
 363       if (result != NULL) return result;
 364     }
 365 #endif
 366
 367 #if HAVE(MMAP)
 368     if (use_mmap && !mmap_failure) {
 369       void* result = TryMmap(size, actual_size, alignment);
 370       if (result != NULL) return result;
 371     }
 372 #endif
 373
 374 #if HAVE(VIRTUALALLOC)
 375     if (use_VirtualAlloc && !VirtualAlloc_failure) {
 376       void* result = TryVirtualAlloc(size, actual_size, alignment);
 377       if (result != NULL) return result;
 378     }
 379 #endif
 380
 381     // nothing worked - reset failure flags and try again
 382     devmem_failure = false;
 383     sbrk_failure = false;
 384     mmap_failure = false;
 385     VirtualAlloc_failure = false;
 386   }
 387   return NULL;
 388 }
 389
 390 #if HAVE(MADV_FREE_REUSE)
 391
 392 void TCMalloc_SystemRelease(void* start, size_t length)
 393 {
 394     while (madvise(start, length, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) { }
 395 }
 396
 397 #elif HAVE(MADV_FREE) || HAVE(MADV_DONTNEED)
 398
 399 void TCMalloc_SystemRelease(void* start, size_t length)
 400 {
 401     // MADV_FREE clears the modified bit on pages, which allows
 402     // them to be discarded immediately.
 403 #if HAVE(MADV_FREE)
 404     const int advice = MADV_FREE;
 405 #else
 406     const int advice = MADV_DONTNEED;
 407 #endif
 408   if (FLAGS_malloc_devmem_start) {
 409     // It's not safe to use MADV_DONTNEED if we've been mapping
 410     // /dev/mem for heap memory
 411     return;
 412   }
 413   if (pagesize == 0) pagesize = getpagesize();
 414   const size_t pagemask = pagesize - 1;
 415
 416   size_t new_start = reinterpret_cast<size_t>(start);
 417   size_t end = new_start + length;
 418   size_t new_end = end;
 419
 420   // Round up the starting address and round down the ending address
 421   // to be page aligned:
 422   new_start = (new_start + pagesize - 1) & ~pagemask;
 423   new_end = new_end & ~pagemask;
 424
 425   ASSERT((new_start & pagemask) == 0);
 426   ASSERT((new_end & pagemask) == 0);
 427   ASSERT(new_start >= reinterpret_cast<size_t>(start));
 428   ASSERT(new_end <= end);
 429
 430   if (new_end > new_start) {
 431     // Note -- ignoring most return codes, because if this fails it
 432     // doesn't matter...
 433     while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
 434                    advice) == -1 &&
 435            errno == EAGAIN) {
 436       // NOP
 437     }
 438   }
 439 }
 440
 441 #elif HAVE(MMAP)
 442
 443 void TCMalloc_SystemRelease(void* start, size_t length)
 444 {
 445   void* newAddress = mmap(start, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
 446   // If the mmap failed then that's ok, we just won't return the memory to the system.
 447   ASSERT_UNUSED(newAddress, newAddress == start || newAddress == reinterpret_cast<void*>(MAP_FAILED));
 448 }
 449
 450 #elif HAVE(VIRTUALALLOC)
 451
 452 void TCMalloc_SystemRelease(void* start, size_t length)
 453 {
 454     if (VirtualFree(start, length, MEM_DECOMMIT))
 455         return;
 456
 457     // The decommit may fail if the memory region consists of allocations
 458     // from more than one call to VirtualAlloc.  In this case, fall back to
 459     // using VirtualQuery to retrieve the allocation boundaries and decommit
 460     // them each individually.
 461
 462     char* ptr = static_cast<char*>(start);
 463     char* end = ptr + length;
 464     MEMORY_BASIC_INFORMATION info;
 465     while (ptr < end) {
 466         size_t resultSize = VirtualQuery(ptr, &info, sizeof(info));
 467         ASSERT_UNUSED(resultSize, resultSize == sizeof(info));
 468
 469         size_t decommitSize = min<size_t>(info.RegionSize, end - ptr);
 470         BOOL success = VirtualFree(ptr, decommitSize, MEM_DECOMMIT);
 471         ASSERT_UNUSED(success, success);
 472         ptr += decommitSize;
 473     }
 474 }
 475
 476 #else
 477
 478 // Platforms that don't support returning memory use an empty inline version of TCMalloc_SystemRelease
 479 // declared in TCSystemAlloc.h
 480
 481 #endif
 482
 483 #if HAVE(MADV_FREE_REUSE)
 484
 485 void TCMalloc_SystemCommit(void* start, size_t length)
 486 {
 487     while (madvise(start, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { }
 488 }
 489
 490 #elif HAVE(VIRTUALALLOC)
 491
 492 void TCMalloc_SystemCommit(void* start, size_t length)
 493 {
 494     if (VirtualAlloc(start, length, MEM_COMMIT, PAGE_READWRITE) == start)
 495         return;
 496
 497     // The commit may fail if the memory region consists of allocations
 498     // from more than one call to VirtualAlloc.  In this case, fall back to
 499     // using VirtualQuery to retrieve the allocation boundaries and commit them
 500     // each individually.
 501
 502     char* ptr = static_cast<char*>(start);
 503     char* end = ptr + length;
 504     MEMORY_BASIC_INFORMATION info;
 505     while (ptr < end) {
 506         size_t resultSize = VirtualQuery(ptr, &info, sizeof(info));
 507         ASSERT_UNUSED(resultSize, resultSize == sizeof(info));
 508
 509         size_t commitSize = min<size_t>(info.RegionSize, end - ptr);
 510         void* newAddress = VirtualAlloc(ptr, commitSize, MEM_COMMIT, PAGE_READWRITE);
 511         ASSERT_UNUSED(newAddress, newAddress == ptr);
 512         ptr += commitSize;
 513     }
 514 }
 515
 516 #else
 517
 518 // Platforms that don't need to explicitly commit memory use an empty inline version of TCMalloc_SystemCommit
 519 // declared in TCSystemAlloc.h
 520
 521 #endif