wtf/TCSystemAlloc.cpp

   1 // Copyright (c) 2005, 2007, Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 //     * Redistributions of source code must retain the above copyright
   9 // notice, this list of conditions and the following disclaimer.
  10 //     * Redistributions in binary form must reproduce the above
  11 // copyright notice, this list of conditions and the following disclaimer
  12 // in the documentation and/or other materials provided with the
  13 // distribution.
  14 //     * Neither the name of Google Inc. nor the names of its
  15 // contributors may be used to endorse or promote products derived from
  16 // this software without specific prior written permission.
  17 //
  18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 // ---
  31 // Author: Sanjay Ghemawat
  32
  33 #include "config.h"
  34 #if HAVE(STDINT_H)
  35 #include <stdint.h>
  36 #elif HAVE(INTTYPES_H)
  37 #include <inttypes.h>
  38 #else
  39 #include <sys/types.h>
  40 #endif
  41 #if PLATFORM(WIN_OS)
  42 #include "windows.h"
  43 #else
  44 #include <errno.h>
  45 #include <unistd.h>
  46 #include <sys/mman.h>
  47 #endif
  48 #include <fcntl.h>
  49 #include "Assertions.h"
  50 #include "TCSystemAlloc.h"
  51 #include "TCSpinLock.h"
  52 #include "UnusedParam.h"
  53
  54 #ifndef MAP_ANONYMOUS
  55 #define MAP_ANONYMOUS MAP_ANON
  56 #endif
  57
  58 // Structure for discovering alignment
  59 union MemoryAligner {
  60   void*  p;
  61   double d;
  62   size_t s;
  63 };
  64
  65 static SpinLock spinlock = SPINLOCK_INITIALIZER;
  66
  67 // Page size is initialized on demand
  68 static size_t pagesize = 0;
  69
  70 // Configuration parameters.
  71 //
  72 // if use_devmem is true, either use_sbrk or use_mmap must also be true.
  73 // For 2.2 kernels, it looks like the sbrk address space (500MBish) and
  74 // the mmap address space (1300MBish) are disjoint, so we need both allocators
  75 // to get as much virtual memory as possible.
  76 #ifndef WTF_CHANGES
  77 static bool use_devmem = false;
  78 #endif
  79
  80 #if HAVE(SBRK)
  81 static bool use_sbrk = false;
  82 #endif
  83
  84 #if HAVE(MMAP)
  85 static bool use_mmap = true;
  86 #endif
  87
  88 #if HAVE(VIRTUALALLOC)
  89 static bool use_VirtualAlloc = true;
  90 #endif
  91
  92 // Flags to keep us from retrying allocators that failed.
  93 static bool devmem_failure = false;
  94 static bool sbrk_failure = false;
  95 static bool mmap_failure = false;
  96 static bool VirtualAlloc_failure = false;
  97
  98 #ifndef WTF_CHANGES
  99 DEFINE_int32(malloc_devmem_start, 0,
 100              "Physical memory starting location in MB for /dev/mem allocation."
 101              "  Setting this to 0 disables /dev/mem allocation");
 102 DEFINE_int32(malloc_devmem_limit, 0,
 103              "Physical memory limit location in MB for /dev/mem allocation."
 104              "  Setting this to 0 means no limit.");
 105 #else
 106 static const int32_t FLAGS_malloc_devmem_start = 0;
 107 static const int32_t FLAGS_malloc_devmem_limit = 0;
 108 #endif
 109
 110 #if HAVE(SBRK)
 111
 112 static void* TrySbrk(size_t size, size_t *actual_size, size_t alignment) {
 113   size = ((size + alignment - 1) / alignment) * alignment;
 114
 115   // could theoretically return the "extra" bytes here, but this
 116   // is simple and correct.
 117   if (actual_size)
 118     *actual_size = size;
 119
 120   void* result = sbrk(size);
 121   if (result == reinterpret_cast<void*>(-1)) {
 122     sbrk_failure = true;
 123     return NULL;
 124   }
 125
 126   // Is it aligned?
 127   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 128   if ((ptr & (alignment-1)) == 0)  return result;
 129
 130   // Try to get more memory for alignment
 131   size_t extra = alignment - (ptr & (alignment-1));
 132   void* r2 = sbrk(extra);
 133   if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
 134     // Contiguous with previous result
 135     return reinterpret_cast<void*>(ptr + extra);
 136   }
 137
 138   // Give up and ask for "size + alignment - 1" bytes so
 139   // that we can find an aligned region within it.
 140   result = sbrk(size + alignment - 1);
 141   if (result == reinterpret_cast<void*>(-1)) {
 142     sbrk_failure = true;
 143     return NULL;
 144   }
 145   ptr = reinterpret_cast<uintptr_t>(result);
 146   if ((ptr & (alignment-1)) != 0) {
 147     ptr += alignment - (ptr & (alignment-1));
 148   }
 149   return reinterpret_cast<void*>(ptr);
 150 }
 151
 152 #endif /* HAVE(SBRK) */
 153
 154 #if HAVE(MMAP)
 155
 156 static void* TryMmap(size_t size, size_t *actual_size, size_t alignment) {
 157   // Enforce page alignment
 158   if (pagesize == 0) pagesize = getpagesize();
 159   if (alignment < pagesize) alignment = pagesize;
 160   size = ((size + alignment - 1) / alignment) * alignment;
 161
 162   // could theoretically return the "extra" bytes here, but this
 163   // is simple and correct.
 164   if (actual_size)
 165     *actual_size = size;
 166
 167   // Ask for extra memory if alignment > pagesize
 168   size_t extra = 0;
 169   if (alignment > pagesize) {
 170     extra = alignment - pagesize;
 171   }
 172   void* result = mmap(NULL, size + extra,
 173                       PROT_READ | PROT_WRITE,
 174                       MAP_PRIVATE|MAP_ANONYMOUS,
 175                       -1, 0);
 176   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
 177     mmap_failure = true;
 178     return NULL;
 179   }
 180
 181   // Adjust the return memory so it is aligned
 182   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 183   size_t adjust = 0;
 184   if ((ptr & (alignment - 1)) != 0) {
 185     adjust = alignment - (ptr & (alignment - 1));
 186   }
 187
 188   // Return the unused memory to the system
 189   if (adjust > 0) {
 190     munmap(reinterpret_cast<void*>(ptr), adjust);
 191   }
 192   if (adjust < extra) {
 193     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
 194   }
 195
 196   ptr += adjust;
 197   return reinterpret_cast<void*>(ptr);
 198 }
 199
 200 #endif /* HAVE(MMAP) */
 201
 202 #if HAVE(VIRTUALALLOC)
 203
 204 static void* TryVirtualAlloc(size_t size, size_t *actual_size, size_t alignment) {
 205   // Enforce page alignment
 206   if (pagesize == 0) {
 207     SYSTEM_INFO system_info;
 208     GetSystemInfo(&system_info);
 209     pagesize = system_info.dwPageSize;
 210   }
 211
 212   if (alignment < pagesize) alignment = pagesize;
 213   size = ((size + alignment - 1) / alignment) * alignment;
 214
 215   // could theoretically return the "extra" bytes here, but this
 216   // is simple and correct.
 217   if (actual_size)
 218     *actual_size = size;
 219
 220   // Ask for extra memory if alignment > pagesize
 221   size_t extra = 0;
 222   if (alignment > pagesize) {
 223     extra = alignment - pagesize;
 224   }
 225   void* result = VirtualAlloc(NULL, size + extra,
 226                               MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN,
 227                               PAGE_READWRITE);
 228
 229   if (result == NULL) {
 230     VirtualAlloc_failure = true;
 231     return NULL;
 232   }
 233
 234   // Adjust the return memory so it is aligned
 235   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 236   size_t adjust = 0;
 237   if ((ptr & (alignment - 1)) != 0) {
 238     adjust = alignment - (ptr & (alignment - 1));
 239   }
 240
 241   // Return the unused memory to the system - we'd like to release but the best we can do
 242   // is decommit, since Windows only lets you free the whole allocation.
 243   if (adjust > 0) {
 244     VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT);
 245   }
 246   if (adjust < extra) {
 247     VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT);
 248   }
 249
 250   ptr += adjust;
 251   return reinterpret_cast<void*>(ptr);
 252 }
 253
 254 #endif /* HAVE(MMAP) */
 255
 256 #ifndef WTF_CHANGES
 257 static void* TryDevMem(size_t size, size_t *actual_size, size_t alignment) {
 258   static bool initialized = false;
 259   static off_t physmem_base;  // next physical memory address to allocate
 260   static off_t physmem_limit; // maximum physical address allowed
 261   static int physmem_fd;      // file descriptor for /dev/mem
 262
 263   // Check if we should use /dev/mem allocation.  Note that it may take
 264   // a while to get this flag initialized, so meanwhile we fall back to
 265   // the next allocator.  (It looks like 7MB gets allocated before
 266   // this flag gets initialized -khr.)
 267   if (FLAGS_malloc_devmem_start == 0) {
 268     // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
 269     // try us again next time.
 270     return NULL;
 271   }
 272
 273   if (!initialized) {
 274     physmem_fd = open("/dev/mem", O_RDWR);
 275     if (physmem_fd < 0) {
 276       devmem_failure = true;
 277       return NULL;
 278     }
 279     physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
 280     physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
 281     initialized = true;
 282   }
 283
 284   // Enforce page alignment
 285   if (pagesize == 0) pagesize = getpagesize();
 286   if (alignment < pagesize) alignment = pagesize;
 287   size = ((size + alignment - 1) / alignment) * alignment;
 288
 289   // could theoretically return the "extra" bytes here, but this
 290   // is simple and correct.
 291   if (actual_size)
 292     *actual_size = size;
 293
 294   // Ask for extra memory if alignment > pagesize
 295   size_t extra = 0;
 296   if (alignment > pagesize) {
 297     extra = alignment - pagesize;
 298   }
 299
 300   // check to see if we have any memory left
 301   if (physmem_limit != 0 && physmem_base + size + extra > physmem_limit) {
 302     devmem_failure = true;
 303     return NULL;
 304   }
 305   void *result = mmap(0, size + extra, PROT_READ | PROT_WRITE,
 306                       MAP_SHARED, physmem_fd, physmem_base);
 307   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
 308     devmem_failure = true;
 309     return NULL;
 310   }
 311   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
 312
 313   // Adjust the return memory so it is aligned
 314   size_t adjust = 0;
 315   if ((ptr & (alignment - 1)) != 0) {
 316     adjust = alignment - (ptr & (alignment - 1));
 317   }
 318
 319   // Return the unused virtual memory to the system
 320   if (adjust > 0) {
 321     munmap(reinterpret_cast<void*>(ptr), adjust);
 322   }
 323   if (adjust < extra) {
 324     munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
 325   }
 326
 327   ptr += adjust;
 328   physmem_base += adjust + size;
 329
 330   return reinterpret_cast<void*>(ptr);
 331 }
 332 #endif
 333
 334 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) {
 335   // Discard requests that overflow
 336   if (size + alignment < size) return NULL;
 337
 338   SpinLockHolder lock_holder(&spinlock);
 339
 340   // Enforce minimum alignment
 341   if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
 342
 343   // Try twice, once avoiding allocators that failed before, and once
 344   // more trying all allocators even if they failed before.
 345   for (int i = 0; i < 2; i++) {
 346
 347 #ifndef WTF_CHANGES
 348     if (use_devmem && !devmem_failure) {
 349       void* result = TryDevMem(size, actual_size, alignment);
 350       if (result != NULL) return result;
 351     }
 352 #endif
 353
 354 #if HAVE(SBRK)
 355     if (use_sbrk && !sbrk_failure) {
 356       void* result = TrySbrk(size, actual_size, alignment);
 357       if (result != NULL) return result;
 358     }
 359 #endif
 360
 361 #if HAVE(MMAP)
 362     if (use_mmap && !mmap_failure) {
 363       void* result = TryMmap(size, actual_size, alignment);
 364       if (result != NULL) return result;
 365     }
 366 #endif
 367
 368 #if HAVE(VIRTUALALLOC)
 369     if (use_VirtualAlloc && !VirtualAlloc_failure) {
 370       void* result = TryVirtualAlloc(size, actual_size, alignment);
 371       if (result != NULL) return result;
 372     }
 373 #endif
 374
 375     // nothing worked - reset failure flags and try again
 376     devmem_failure = false;
 377     sbrk_failure = false;
 378     mmap_failure = false;
 379     VirtualAlloc_failure = false;
 380   }
 381   return NULL;
 382 }
 383
 384 void TCMalloc_SystemRelease(void* start, size_t length)
 385 {
 386 #if HAVE(MADV_FREE) || HAVE(MADV_DONTNEED)
 387 #if HAVE(MADV_FREE)
 388   const int advice = MADV_FREE;
 389 #else
 390   const int advice = MADV_DONTNEED;
 391 #endif
 392   if (FLAGS_malloc_devmem_start) {
 393     // It's not safe to use MADV_DONTNEED if we've been mapping
 394     // /dev/mem for heap memory
 395     return;
 396   }
 397   if (pagesize == 0) pagesize = getpagesize();
 398   const size_t pagemask = pagesize - 1;
 399
 400   size_t new_start = reinterpret_cast<size_t>(start);
 401   size_t end = new_start + length;
 402   size_t new_end = end;
 403
 404   // Round up the starting address and round down the ending address
 405   // to be page aligned:
 406   new_start = (new_start + pagesize - 1) & ~pagemask;
 407   new_end = new_end & ~pagemask;
 408
 409   ASSERT((new_start & pagemask) == 0);
 410   ASSERT((new_end & pagemask) == 0);
 411   ASSERT(new_start >= reinterpret_cast<size_t>(start));
 412   ASSERT(new_end <= end);
 413
 414   if (new_end > new_start) {
 415     // Note -- ignoring most return codes, because if this fails it
 416     // doesn't matter...
 417     while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
 418                    advice) == -1 &&
 419            errno == EAGAIN) {
 420       // NOP
 421     }
 422     return;
 423   }
 424 #endif
 425
 426 #if HAVE(MMAP)
 427   void* newAddress = mmap(start, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
 428   // If the mmap failed then that's ok, we just won't return the memory to the system.
 429   ASSERT_UNUSED(newAddress, newAddress == start || newAddress == reinterpret_cast<void*>(MAP_FAILED));
 430   return;
 431 #endif
 432
 433 #if !HAVE(MADV_DONTNEED) && !HAVE(MMAP)
 434   UNUSED_PARAM(start);
 435   UNUSED_PARAM(length);
 436 #endif
 437 }
 438
 439 #if HAVE(VIRTUALALLOC)
 440 void TCMalloc_SystemCommit(void*, size_t)
 441 {
 442 }
 443 #endif