X-Git-Url: https://git.saurik.com/apple/javascriptcore.git/blobdiff_plain/b37bf2e156556c589aea3e1f58a377f2b1189665..14957cd040308e3eeec43d26bae5d76da13fcd85:/wtf/FastMalloc.cpp?ds=sidebyside

diff --git a/wtf/FastMalloc.cpp b/wtf/FastMalloc.cpp
index 7d17907..c42c3c4 100644
--- a/wtf/FastMalloc.cpp
+++ b/wtf/FastMalloc.cpp
@@ -1,5 +1,6 @@
 // Copyright (c) 2005, 2007, Google Inc.
 // All rights reserved.
+// Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserved.
 // 
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
@@ -77,9 +78,11 @@
 #include "FastMalloc.h"
 
 #include "Assertions.h"
-#if USE(MULTIPLE_THREADS)
+#include <limits>
+#if ENABLE(WTF_MULTIPLE_THREADS)
 #include <pthread.h>
 #endif
+#include <wtf/StdLibExtras.h>
 
 #ifndef NO_TCMALLOC_SAMPLES
 #ifdef WTF_CHANGES
@@ -87,16 +90,19 @@
 #endif
 #endif
 
-#if !defined(USE_SYSTEM_MALLOC) && defined(NDEBUG)
+#if !(defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC) && defined(NDEBUG)
 #define FORCE_SYSTEM_MALLOC 0
 #else
 #define FORCE_SYSTEM_MALLOC 1
 #endif
 
+// Use a background thread to periodically scavenge memory to release back to the system
+#define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 0
+
 #ifndef NDEBUG
 namespace WTF {
 
-#if USE(MULTIPLE_THREADS)
+#if ENABLE(WTF_MULTIPLE_THREADS)
 static pthread_key_t isForbiddenKey;
 static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT;
 static void initializeIsForbiddenKey()
@@ -104,11 +110,13 @@ static void initializeIsForbiddenKey()
   pthread_key_create(&isForbiddenKey, 0);
 }
 
+#if !ASSERT_DISABLED
 static bool isForbidden()
 {
     pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey);
     return !!pthread_getspecific(isForbiddenKey);
 }
+#endif
 
 void fastMallocForbid()
 {
@@ -139,7 +147,7 @@ void fastMallocAllow()
 {
     staticIsForbidden = false;
 }
-#endif // USE(MULTIPLE_THREADS)
+#endif // ENABLE(WTF_MULTIPLE_THREADS)
 
 } // namespace WTF
 #endif // NDEBUG
@@ -147,66 +155,250 @@ void fastMallocAllow()
 #include <string.h>
 
 namespace WTF {
-void *fastZeroedMalloc(size_t n) 
+
+
+namespace Internal {
+#if !ENABLE(WTF_MALLOC_VALIDATION)
+void fastMallocMatchFailed(void*);
+#else
+COMPILE_ASSERT(((sizeof(ValidationHeader) % sizeof(AllocAlignmentInteger)) == 0), ValidationHeader_must_produce_correct_alignment);
+#endif
+void fastMallocMatchFailed(void*)
 {
-    void *result = fastMalloc(n);
-    if (!result)
-        return 0;
+    CRASH();
+}
+
+} // namespace Internal
+
+
+void* fastZeroedMalloc(size_t n) 
+{
+    void* result = fastMalloc(n);
     memset(result, 0, n);
-#ifndef WTF_CHANGES
-    MallocHook::InvokeNewHook(result, n);
-#endif
     return result;
 }
-    
+
+char* fastStrDup(const char* src)
+{
+    size_t len = strlen(src) + 1;
+    char* dup = static_cast<char*>(fastMalloc(len));
+    memcpy(dup, src, len);
+    return dup;
 }
 
+TryMallocReturnValue tryFastZeroedMalloc(size_t n) 
+{
+    void* result;
+    if (!tryFastMalloc(n).getValue(result))
+        return 0;
+    memset(result, 0, n);
+    return result;
+}
+
+} // namespace WTF
+
 #if FORCE_SYSTEM_MALLOC
 
-#include <stdlib.h>
-#if !PLATFORM(WIN_OS)
-    #include <pthread.h>
+#if PLATFORM(BREWMP)
+#include "brew/SystemMallocBrew.h"
+#endif
+
+#if OS(DARWIN)
+#include <malloc/malloc.h>
+#elif OS(WINDOWS)
+#include <malloc.h>
 #endif
 
 namespace WTF {
-    
-void *fastMalloc(size_t n) 
+
+TryMallocReturnValue tryFastMalloc(size_t n) 
 {
     ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    if (std::numeric_limits<size_t>::max() - Internal::ValidationBufferSize <= n)  // If overflow would occur...
+        return 0;
+
+    void* result = malloc(n + Internal::ValidationBufferSize);
+    if (!result)
+        return 0;
+    Internal::ValidationHeader* header = static_cast<Internal::ValidationHeader*>(result);
+    header->m_size = n;
+    header->m_type = Internal::AllocTypeMalloc;
+    header->m_prefix = static_cast<unsigned>(Internal::ValidationPrefix);
+    result = header + 1;
+    *Internal::fastMallocValidationSuffix(result) = Internal::ValidationSuffix;
+    fastMallocValidate(result);
+    return result;
+#else
     return malloc(n);
+#endif
 }
 
-void *fastCalloc(size_t n_elements, size_t element_size)
+void* fastMalloc(size_t n) 
 {
     ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    TryMallocReturnValue returnValue = tryFastMalloc(n);
+    void* result;
+    if (!returnValue.getValue(result))
+        CRASH();
+#else
+    void* result = malloc(n);
+#endif
+
+    if (!result) {
+#if PLATFORM(BREWMP)
+        // The behavior of malloc(0) is implementation defined.
+        // To make sure that fastMalloc never returns 0, retry with fastMalloc(1).
+        if (!n)
+            return fastMalloc(1);
+#endif
+        CRASH();
+    }
+
+    return result;
+}
+
+TryMallocReturnValue tryFastCalloc(size_t n_elements, size_t element_size)
+{
+    ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    size_t totalBytes = n_elements * element_size;
+    if (n_elements > 1 && element_size && (totalBytes / element_size) != n_elements)
+        return 0;
+
+    TryMallocReturnValue returnValue = tryFastMalloc(totalBytes);
+    void* result;
+    if (!returnValue.getValue(result))
+        return 0;
+    memset(result, 0, totalBytes);
+    fastMallocValidate(result);
+    return result;
+#else
     return calloc(n_elements, element_size);
+#endif
+}
+
+void* fastCalloc(size_t n_elements, size_t element_size)
+{
+    ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    TryMallocReturnValue returnValue = tryFastCalloc(n_elements, element_size);
+    void* result;
+    if (!returnValue.getValue(result))
+        CRASH();
+#else
+    void* result = calloc(n_elements, element_size);
+#endif
+
+    if (!result) {
+#if PLATFORM(BREWMP)
+        // If either n_elements or element_size is 0, the behavior of calloc is implementation defined.
+        // To make sure that fastCalloc never returns 0, retry with fastCalloc(1, 1).
+        if (!n_elements || !element_size)
+            return fastCalloc(1, 1);
+#endif
+        CRASH();
+    }
+
+    return result;
 }
 
 void fastFree(void* p)
 {
     ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    if (!p)
+        return;
+    
+    fastMallocMatchValidateFree(p, Internal::AllocTypeMalloc);
+    Internal::ValidationHeader* header = Internal::fastMallocValidationHeader(p);
+    memset(p, 0xCC, header->m_size);
+    free(header);
+#else
     free(p);
+#endif
 }
 
-void *fastRealloc(void* p, size_t n)
+TryMallocReturnValue tryFastRealloc(void* p, size_t n)
 {
     ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    if (p) {
+        if (std::numeric_limits<size_t>::max() - Internal::ValidationBufferSize <= n)  // If overflow would occur...
+            return 0;
+        fastMallocValidate(p);
+        Internal::ValidationHeader* result = static_cast<Internal::ValidationHeader*>(realloc(Internal::fastMallocValidationHeader(p), n + Internal::ValidationBufferSize));
+        if (!result)
+            return 0;
+        result->m_size = n;
+        result = result + 1;
+        *fastMallocValidationSuffix(result) = Internal::ValidationSuffix;
+        fastMallocValidate(result);
+        return result;
+    } else {
+        return fastMalloc(n);
+    }
+#else
     return realloc(p, n);
+#endif
 }
 
-} // namespace WTF
+void* fastRealloc(void* p, size_t n)
+{
+    ASSERT(!isForbidden());
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    TryMallocReturnValue returnValue = tryFastRealloc(p, n);
+    void* result;
+    if (!returnValue.getValue(result))
+        CRASH();
+#else
+    void* result = realloc(p, n);
+#endif
+
+    if (!result)
+        CRASH();
+    return result;
+}
 
-extern "C" {
 void releaseFastMallocFreeMemory() { }
+    
+FastMallocStatistics fastMallocStatistics()
+{
+    FastMallocStatistics statistics = { 0, 0, 0 };
+    return statistics;
+}
+
+size_t fastMallocSize(const void* p)
+{
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    return Internal::fastMallocValidationHeader(const_cast<void*>(p))->m_size;
+#elif OS(DARWIN)
+    return malloc_size(p);
+#elif OS(WINDOWS) && !PLATFORM(BREWMP)
+    // Brew MP uses its own memory allocator, so _msize does not work on the Brew MP simulator.
+    return _msize(const_cast<void*>(p));
+#else
+    return 1;
+#endif
 }
 
-#if PLATFORM(DARWIN)
+} // namespace WTF
+
+#if OS(DARWIN)
 // This symbol is present in the JavaScriptCore exports file even when FastMalloc is disabled.
 // It will never be used in this case, so it's type and value are less interesting than its presence.
 extern "C" const int jscore_fastmalloc_introspection = 0;
 #endif
 
-#else
+#else // FORCE_SYSTEM_MALLOC
 
 #if HAVE(STDINT_H)
 #include <stdint.h>
@@ -223,23 +415,46 @@ extern "C" const int jscore_fastmalloc_introspection = 0;
 #include "TCSpinLock.h"
 #include "TCSystemAlloc.h"
 #include <algorithm>
-#include <errno.h>
-#include <new>
+#include <limits>
 #include <pthread.h>
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdio.h>
-#if COMPILER(MSVC)
+#if HAVE(ERRNO_H)
+#include <errno.h>
+#endif
+#if OS(UNIX)
+#include <unistd.h>
+#endif
+#if OS(WINDOWS)
 #ifndef WIN32_LEAN_AND_MEAN
 #define WIN32_LEAN_AND_MEAN
 #endif
 #include <windows.h>
 #endif
 
-#if WTF_CHANGES
+#ifdef WTF_CHANGES
 
-#if PLATFORM(DARWIN)
+#if OS(DARWIN)
 #include "MallocZoneSupport.h"
+#include <wtf/HashSet.h>
+#include <wtf/Vector.h>
+#endif
+
+#if HAVE(HEADER_DETECTION_H)
+#include "HeaderDetection.h"
+#endif
+
+#if HAVE(DISPATCH_H)
+#include <dispatch/dispatch.h>
+#endif
+
+#if HAVE(PTHREAD_MACHDEP_H)
+#include <System/pthread_machdep.h>
+
+#if defined(__PTK_FRAMEWORK_JAVASCRIPTCORE_KEY0)
+#define WTF_USE_PTHREAD_GETSPECIFIC_DIRECT 1
+#endif
 #endif
 
 #ifndef PRIuS
@@ -250,9 +465,14 @@ extern "C" const int jscore_fastmalloc_introspection = 0;
 // call to the function on Mac OS X, and it's used in performance-critical code. So we
 // use a function pointer. But that's not necessarily faster on other platforms, and we had
 // problems with this technique on Windows, so we'll do this only on Mac OS X.
-#if PLATFORM(DARWIN)
+#if OS(DARWIN)
+#if !USE(PTHREAD_GETSPECIFIC_DIRECT)
 static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_getspecific;
 #define pthread_getspecific(key) pthread_getspecific_function_pointer(key)
+#else
+#define pthread_getspecific(key) _pthread_getspecific_direct(key)
+#define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val))
+#endif
 #endif
 
 #define DEFINE_VARIABLE(type, name, value, meaning) \
@@ -278,10 +498,12 @@ namespace WTF {
 #define MESSAGE LOG_ERROR
 #define CHECK_CONDITION ASSERT
 
-#if PLATFORM(DARWIN)
+#if OS(DARWIN)
+struct Span;
+class TCMalloc_Central_FreeListPadded;
 class TCMalloc_PageHeap;
 class TCMalloc_ThreadCache;
-class TCMalloc_Central_FreeListPadded;
+template <typename T> class PageHeapAllocator;
 
 class FastMallocZone {
 public:
@@ -294,10 +516,10 @@ public:
     static void log(malloc_zone_t*, void*) { }
     static void forceLock(malloc_zone_t*) { }
     static void forceUnlock(malloc_zone_t*) { }
-    static void statistics(malloc_zone_t*, malloc_statistics_t*) { }
+    static void statistics(malloc_zone_t*, malloc_statistics_t* stats) { memset(stats, 0, sizeof(malloc_statistics_t)); }
 
 private:
-    FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*);
+    FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*, PageHeapAllocator<Span>*, PageHeapAllocator<TCMalloc_ThreadCache>*);
     static size_t size(malloc_zone_t*, const void*);
     static void* zoneMalloc(malloc_zone_t*, size_t);
     static void* zoneCalloc(malloc_zone_t*, size_t numItems, size_t size);
@@ -310,6 +532,8 @@ private:
     TCMalloc_PageHeap* m_pageHeap;
     TCMalloc_ThreadCache** m_threadHeaps;
     TCMalloc_Central_FreeListPadded* m_centralCaches;
+    PageHeapAllocator<Span>* m_spanAllocator;
+    PageHeapAllocator<TCMalloc_ThreadCache>* m_pageHeapAllocator;
 };
 
 #endif
@@ -390,7 +614,7 @@ static const size_t kNumClasses = 68;
 static const size_t kPageMapBigAllocationThreshold = 128 << 20;
 
 // Minimum number of pages to fetch from system at a time.  Must be
-// significantly bigger than kBlockSize to amortize system-call
+// significantly bigger than kPageSize to amortize system-call
 // overhead, and also to reduce external fragementation.  Also, we
 // should keep this value big because various incarnations of Linux
 // have small limits on the number of mmap() regions per
@@ -413,7 +637,7 @@ static const int kMaxFreeListLength = 256;
 
 // Lower and upper bounds on the per-thread cache sizes
 static const size_t kMinThreadCacheSize = kMaxSize * 2;
-static const size_t kMaxThreadCacheSize = 2 << 20;
+static const size_t kMaxThreadCacheSize = 512 * 1024;
 
 // Default bound on the total amount of thread caches
 static const size_t kDefaultOverallThreadCacheSize = 16 << 20;
@@ -628,11 +852,11 @@ static void InitSizeClasses() {
   // Do some sanity checking on add_amount[]/shift_amount[]/class_array[]
   if (ClassIndex(0) < 0) {
     MESSAGE("Invalid class index %d for size 0\n", ClassIndex(0));
-    abort();
+    CRASH();
   }
   if (static_cast<size_t>(ClassIndex(kMaxSize)) >= sizeof(class_array)) {
     MESSAGE("Invalid class index %d for kMaxSize\n", ClassIndex(kMaxSize));
-    abort();
+    CRASH();
   }
 
   // Compute the size classes we want to use
@@ -684,7 +908,7 @@ static void InitSizeClasses() {
   if (sc != kNumClasses) {
     MESSAGE("wrong number of size classes: found %" PRIuS " instead of %d\n",
             sc, int(kNumClasses));
-    abort();
+    CRASH();
   }
 
   // Initialize the mapping arrays
@@ -702,25 +926,25 @@ static void InitSizeClasses() {
     const size_t sc = SizeClass(size);
     if (sc == 0) {
       MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size);
-      abort();
+      CRASH();
     }
     if (sc > 1 && size <= class_to_size[sc-1]) {
       MESSAGE("Allocating unnecessarily large class %" PRIuS " for %" PRIuS
               "\n", sc, size);
-      abort();
+      CRASH();
     }
     if (sc >= kNumClasses) {
       MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size);
-      abort();
+      CRASH();
     }
     const size_t s = class_to_size[sc];
     if (size > s) {
      MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc);
-      abort();
+      CRASH();
     }
     if (s == 0) {
       MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc);
-      abort();
+      CRASH();
     }
   }
 
@@ -778,6 +1002,9 @@ class PageHeapAllocator {
   char* free_area_;
   size_t free_avail_;
 
+  // Linked list of all regions allocated by this allocator
+  void* allocated_regions_;
+
   // Free list of already carved objects
   void* free_list_;
 
@@ -788,6 +1015,7 @@ class PageHeapAllocator {
   void Init() {
     ASSERT(kAlignedSize <= kAllocIncrement);
     inuse_ = 0;
+    allocated_regions_ = 0;
     free_area_ = NULL;
     free_avail_ = 0;
     free_list_ = NULL;
@@ -802,9 +1030,14 @@ class PageHeapAllocator {
     } else {
       if (free_avail_ < kAlignedSize) {
         // Need more room
-        free_area_ = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));
-        if (free_area_ == NULL) abort();
-        free_avail_ = kAllocIncrement;
+        char* new_allocation = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));
+        if (!new_allocation)
+          CRASH();
+
+        *reinterpret_cast_ptr<void**>(new_allocation) = allocated_regions_;
+        allocated_regions_ = new_allocation;
+        free_area_ = new_allocation + kAlignedSize;
+        free_avail_ = kAllocIncrement - kAlignedSize;
       }
       result = free_area_;
       free_area_ += kAlignedSize;
@@ -821,6 +1054,15 @@ class PageHeapAllocator {
   }
 
   int inuse() const { return inuse_; }
+
+#if defined(WTF_CHANGES) && OS(DARWIN)
+  template <class Recorder>
+  void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader& reader)
+  {
+      for (void* adminAllocation = allocated_regions_; adminAllocation; adminAllocation = reader.nextEntryInLinkedList(reinterpret_cast<void**>(adminAllocation)))
+          recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation), kAllocIncrement);
+  }
+#endif
 };
 
 // -------------------------------------------------------------------------
@@ -863,9 +1105,12 @@ struct Span {
   Span*         prev;           // Used when in link list
   void*         objects;        // Linked list of free objects
   unsigned int  free : 1;       // Is the span free
+#ifndef NO_TCMALLOC_SAMPLES
   unsigned int  sample : 1;     // Sampled object?
+#endif
   unsigned int  sizeclass : 8;  // Size-class for small objects (or 0)
   unsigned int  refcount : 11;  // Number of non-free objects
+  bool decommitted : 1;
 
 #undef SPAN_HISTORY
 #ifdef SPAN_HISTORY
@@ -876,6 +1121,8 @@ struct Span {
 #endif
 };
 
+#define ASSERT_SPAN_COMMITTED(span) ASSERT(!span->decommitted)
+
 #ifdef SPAN_HISTORY
 void Event(Span* span, char op, int v = 0) {
   span->history[span->nexthistory] = op;
@@ -928,7 +1175,6 @@ static ALWAYS_INLINE bool DLL_IsEmpty(const Span* list) {
   return list->next == list;
 }
 
-#ifndef WTF_CHANGES
 static int DLL_Length(const Span* list) {
   int result = 0;
   for (Span* s = list->next; s != list; s = s->next) {
@@ -936,7 +1182,6 @@ static int DLL_Length(const Span* list) {
   }
   return result;
 }
-#endif
 
 #if 0 /* Not needed at the moment -- causes compiler warnings if not used */
 static void DLL_Print(const char* label, const Span* list) {
@@ -988,11 +1233,30 @@ template <int BITS> class MapSelector {
   typedef PackedCache<BITS, uint64_t> CacheType;
 };
 
+#if defined(WTF_CHANGES)
+#if CPU(X86_64)
+// On all known X86-64 platforms, the upper 16 bits are always unused and therefore 
+// can be excluded from the PageMap key.
+// See http://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details
+
+static const size_t kBitsUnusedOn64Bit = 16;
+#else
+static const size_t kBitsUnusedOn64Bit = 0;
+#endif
+
+// A three-level map for 64-bit machines
+template <> class MapSelector<64> {
+ public:
+  typedef TCMalloc_PageMap3<64 - kPageShift - kBitsUnusedOn64Bit> Type;
+  typedef PackedCache<64, uint64_t> CacheType;
+};
+#endif
+
 // A two-level map for 32-bit machines
 template <> class MapSelector<32> {
  public:
-  typedef TCMalloc_PageMap2<32-kPageShift> Type;
-  typedef PackedCache<32-kPageShift, uint16_t> CacheType;
+  typedef TCMalloc_PageMap2<32 - kPageShift> Type;
+  typedef PackedCache<32 - kPageShift, uint16_t> CacheType;
 };
 
 // -------------------------------------------------------------------------
@@ -1003,6 +1267,39 @@ template <> class MapSelector<32> {
 // contiguous runs of pages (called a "span").
 // -------------------------------------------------------------------------
 
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+// The page heap maintains a free list for spans that are no longer in use by
+// the central cache or any thread caches. We use a background thread to
+// periodically scan the free list and release a percentage of it back to the OS.
+
+// If free_committed_pages_ exceeds kMinimumFreeCommittedPageCount, the
+// background thread:
+//     - wakes up
+//     - pauses for kScavengeDelayInSeconds
+//     - returns to the OS a percentage of the memory that remained unused during
+//       that pause (kScavengePercentage * min_free_committed_pages_since_last_scavenge_)
+// The goal of this strategy is to reduce memory pressure in a timely fashion
+// while avoiding thrashing the OS allocator.
+
+// Time delay before the page heap scavenger will consider returning pages to
+// the OS.
+static const int kScavengeDelayInSeconds = 2;
+
+// Approximate percentage of free committed pages to return to the OS in one
+// scavenge.
+static const float kScavengePercentage = .5f;
+
+// number of span lists to keep spans in when memory is returned.
+static const int kMinSpanListsWithSpans = 32;
+
+// Number of free committed pages that we want to keep around.  The minimum number of pages used when there
+// is 1 span in each of the first kMinSpanListsWithSpans spanlists.  Currently 528 pages.
+static const size_t kMinimumFreeCommittedPageCount = kMinSpanListsWithSpans * ((1.0f+kMinSpanListsWithSpans) / 2.0f);
+
+#endif
+
+static SpinLock pageheap_lock = SPINLOCK_INITIALIZER;
+
 class TCMalloc_PageHeap {
  public:
   void init();
@@ -1042,6 +1339,8 @@ class TCMalloc_PageHeap {
       pagemap_.Ensure(p, 1);
       return GetDescriptor(p);
   }
+    
+  size_t ReturnedBytes() const;
 #endif
 
   // Dump state to stderr
@@ -1058,7 +1357,7 @@ class TCMalloc_PageHeap {
   }
 
   bool Check();
-  bool CheckList(Span* list, Length min_pages, Length max_pages);
+  bool CheckList(Span* list, Length min_pages, Length max_pages, bool decommitted);
 
   // Release all pages on the free list for reuse by the OS:
   void ReleaseFreePages();
@@ -1100,6 +1399,15 @@ class TCMalloc_PageHeap {
   // Bytes allocated from system
   uint64_t system_bytes_;
 
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+  // Number of pages kept in free lists that are still committed.
+  Length free_committed_pages_;
+
+  // Minimum number of free committed pages since last scavenge. (Can be 0 if
+  // we've committed new pages since the last scavenge.)
+  Length min_free_committed_pages_since_last_scavenge_;
+#endif
+
   bool GrowHeap(Length n);
 
   // REQUIRES   span->length >= n
@@ -1122,9 +1430,11 @@ class TCMalloc_PageHeap {
   // span of exactly the specified length.  Else, returns NULL.
   Span* AllocLarge(Length n);
 
+#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
   // Incrementally release some memory to the system.
   // IncrementalScavenge(n) is called whenever n pages are freed.
   void IncrementalScavenge(Length n);
+#endif
 
   // Number of pages to deallocate before doing more scavenging
   int64_t scavenge_counter_;
@@ -1132,9 +1442,44 @@ class TCMalloc_PageHeap {
   // Index of last free list we scavenged
   size_t scavenge_index_;
   
-#if defined(WTF_CHANGES) && PLATFORM(DARWIN)
+#if defined(WTF_CHANGES) && OS(DARWIN)
   friend class FastMallocZone;
 #endif
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+  void initializeScavenger();
+  ALWAYS_INLINE void signalScavenger();
+  void scavenge();
+  ALWAYS_INLINE bool shouldScavenge() const;
+
+#if HAVE(DISPATCH_H) || OS(WINDOWS)
+  void periodicScavenge();
+  ALWAYS_INLINE bool isScavengerSuspended();
+  ALWAYS_INLINE void scheduleScavenger();
+  ALWAYS_INLINE void rescheduleScavenger();
+  ALWAYS_INLINE void suspendScavenger();
+#endif
+
+#if HAVE(DISPATCH_H)
+  dispatch_queue_t m_scavengeQueue;
+  dispatch_source_t m_scavengeTimer;
+  bool m_scavengingSuspended;
+#elif OS(WINDOWS)
+  static void CALLBACK scavengerTimerFired(void*, BOOLEAN);
+  HANDLE m_scavengeQueueTimer;
+#else 
+  static NO_RETURN_WITH_VALUE void* runScavengerThread(void*);
+  NO_RETURN void scavengerThread();
+
+  // Keeps track of whether the background thread is actively scavenging memory every kScavengeDelayInSeconds, or
+  // it's blocked waiting for more pages to be deleted.
+  bool m_scavengeThreadActive;
+
+  pthread_mutex_t m_scavengeMutex;
+  pthread_cond_t m_scavengeCondition;
+#endif
+
+#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
 };
 
 void TCMalloc_PageHeap::init()
@@ -1143,6 +1488,12 @@ void TCMalloc_PageHeap::init()
   pagemap_cache_ = PageMapCache(0);
   free_pages_ = 0;
   system_bytes_ = 0;
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+  free_committed_pages_ = 0;
+  min_free_committed_pages_since_last_scavenge_ = 0;
+#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
   scavenge_counter_ = 0;
   // Start scavenging at kMaxPages list
   scavenge_index_ = kMaxPages-1;
@@ -1153,8 +1504,178 @@ void TCMalloc_PageHeap::init()
     DLL_Init(&free_[i].normal);
     DLL_Init(&free_[i].returned);
   }
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+  initializeScavenger();
+#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+}
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
+#if HAVE(DISPATCH_H)
+
+void TCMalloc_PageHeap::initializeScavenger()
+{
+    m_scavengeQueue = dispatch_queue_create("com.apple.JavaScriptCore.FastMallocSavenger", NULL);
+    m_scavengeTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, m_scavengeQueue);
+    dispatch_time_t startTime = dispatch_time(DISPATCH_TIME_NOW, kScavengeDelayInSeconds * NSEC_PER_SEC);
+    dispatch_source_set_timer(m_scavengeTimer, startTime, kScavengeDelayInSeconds * NSEC_PER_SEC, 1000 * NSEC_PER_USEC);
+    dispatch_source_set_event_handler(m_scavengeTimer, ^{ periodicScavenge(); });
+    m_scavengingSuspended = true;
+}
+
+ALWAYS_INLINE bool TCMalloc_PageHeap::isScavengerSuspended()
+{
+    ASSERT(pageheap_lock.IsHeld());
+    return m_scavengingSuspended;
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::scheduleScavenger()
+{
+    ASSERT(pageheap_lock.IsHeld());
+    m_scavengingSuspended = false;
+    dispatch_resume(m_scavengeTimer);
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::rescheduleScavenger()
+{
+    // Nothing to do here for libdispatch.
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger()
+{
+    ASSERT(pageheap_lock.IsHeld());
+    m_scavengingSuspended = true;
+    dispatch_suspend(m_scavengeTimer);
+}
+
+#elif OS(WINDOWS)
+
+void TCMalloc_PageHeap::scavengerTimerFired(void* context, BOOLEAN)
+{
+    static_cast<TCMalloc_PageHeap*>(context)->periodicScavenge();
+}
+
+void TCMalloc_PageHeap::initializeScavenger()
+{
+    m_scavengeQueueTimer = 0;
+}
+
+ALWAYS_INLINE bool TCMalloc_PageHeap::isScavengerSuspended()
+{
+    ASSERT(IsHeld(pageheap_lock));
+    return !m_scavengeQueueTimer;
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::scheduleScavenger()
+{
+    // We need to use WT_EXECUTEONLYONCE here and reschedule the timer, because
+    // Windows will fire the timer event even when the function is already running.
+    ASSERT(IsHeld(pageheap_lock));
+    CreateTimerQueueTimer(&m_scavengeQueueTimer, 0, scavengerTimerFired, this, kScavengeDelayInSeconds * 1000, 0, WT_EXECUTEONLYONCE);
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::rescheduleScavenger()
+{
+    // We must delete the timer and create it again, because it is not possible to retrigger a timer on Windows.
+    suspendScavenger();
+    scheduleScavenger();
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger()
+{
+    ASSERT(IsHeld(pageheap_lock));
+    HANDLE scavengeQueueTimer = m_scavengeQueueTimer;
+    m_scavengeQueueTimer = 0;
+    DeleteTimerQueueTimer(0, scavengeQueueTimer, 0);
+}
+
+#else
+
+void TCMalloc_PageHeap::initializeScavenger()
+{
+    // Create a non-recursive mutex.
+#if !defined(PTHREAD_MUTEX_NORMAL) || PTHREAD_MUTEX_NORMAL == PTHREAD_MUTEX_DEFAULT
+    pthread_mutex_init(&m_scavengeMutex, 0);
+#else
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL);
+
+    pthread_mutex_init(&m_scavengeMutex, &attr);
+
+    pthread_mutexattr_destroy(&attr);
+#endif
+
+    pthread_cond_init(&m_scavengeCondition, 0);
+    m_scavengeThreadActive = true;
+    pthread_t thread;
+    pthread_create(&thread, 0, runScavengerThread, this);
+}
+
+void* TCMalloc_PageHeap::runScavengerThread(void* context)
+{
+    static_cast<TCMalloc_PageHeap*>(context)->scavengerThread();
+#if (COMPILER(MSVC) || COMPILER(SUNCC))
+    // Without this, Visual Studio and Sun Studio will complain that this method does not return a value.
+    return 0;
+#endif
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger()
+{
+    // m_scavengeMutex should be held before accessing m_scavengeThreadActive.
+    ASSERT(pthread_mutex_trylock(m_scavengeMutex));
+    if (!m_scavengeThreadActive && shouldScavenge())
+        pthread_cond_signal(&m_scavengeCondition);
+}
+
+#endif
+
+void TCMalloc_PageHeap::scavenge()
+{
+    size_t pagesToRelease = min_free_committed_pages_since_last_scavenge_ * kScavengePercentage;
+    size_t targetPageCount = std::max<size_t>(kMinimumFreeCommittedPageCount, free_committed_pages_ - pagesToRelease);
+
+    Length lastFreeCommittedPages = free_committed_pages_;
+    while (free_committed_pages_ > targetPageCount) {
+        ASSERT(Check());
+        for (int i = kMaxPages; i > 0 && free_committed_pages_ >= targetPageCount; i--) {
+            SpanList* slist = (static_cast<size_t>(i) == kMaxPages) ? &large_ : &free_[i];
+            // If the span size is bigger than kMinSpanListsWithSpans pages return all the spans in the list, else return all but 1 span.  
+            // Return only 50% of a spanlist at a time so spans of size 1 are not the only ones left.
+            size_t length = DLL_Length(&slist->normal);
+            size_t numSpansToReturn = (i > kMinSpanListsWithSpans) ? length : length / 2;
+            for (int j = 0; static_cast<size_t>(j) < numSpansToReturn && !DLL_IsEmpty(&slist->normal) && free_committed_pages_ > targetPageCount; j++) {
+                Span* s = slist->normal.prev; 
+                DLL_Remove(s);
+                ASSERT(!s->decommitted);
+                if (!s->decommitted) {
+                    TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
+                                           static_cast<size_t>(s->length << kPageShift));
+                    ASSERT(free_committed_pages_ >= s->length);
+                    free_committed_pages_ -= s->length;
+                    s->decommitted = true;
+                }
+                DLL_Prepend(&slist->returned, s);
+            }
+        }
+
+        if (lastFreeCommittedPages == free_committed_pages_)
+            break;
+        lastFreeCommittedPages = free_committed_pages_;
+    }
+
+    min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
 }
 
+ALWAYS_INLINE bool TCMalloc_PageHeap::shouldScavenge() const 
+{
+    return free_committed_pages_ > kMinimumFreeCommittedPageCount; 
+}
+
+#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
 inline Span* TCMalloc_PageHeap::New(Length n) {
   ASSERT(Check());
   ASSERT(n > 0);
@@ -1177,13 +1698,24 @@ inline Span* TCMalloc_PageHeap::New(Length n) {
 
     Span* result = ll->next;
     Carve(result, n, released);
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+    // The newly allocated memory is from a span that's in the normal span list (already committed).  Update the
+    // free committed pages count.
+    ASSERT(free_committed_pages_ >= n);
+    free_committed_pages_ -= n;
+    if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) 
+      min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
     ASSERT(Check());
     free_pages_ -= n;
     return result;
   }
 
   Span* result = AllocLarge(n);
-  if (result != NULL) return result;
+  if (result != NULL) {
+      ASSERT_SPAN_COMMITTED(result);
+      return result;
+  }
 
   // Grow the heap and try again
   if (!GrowHeap(n)) {
@@ -1230,6 +1762,14 @@ Span* TCMalloc_PageHeap::AllocLarge(Length n) {
 
   if (best != NULL) {
     Carve(best, n, from_released);
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+    // The newly allocated memory is from a span that's in the normal span list (already committed).  Update the
+    // free committed pages count.
+    ASSERT(free_committed_pages_ >= n);
+    free_committed_pages_ -= n;
+    if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_)
+      min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+#endif  // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
     ASSERT(Check());
     free_pages_ -= n;
     return best;
@@ -1260,17 +1800,28 @@ inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) {
   span->free = 0;
   Event(span, 'A', n);
 
+  if (released) {
+    // If the span chosen to carve from is decommited, commit the entire span at once to avoid committing spans 1 page at a time.
+    ASSERT(span->decommitted);
+    TCMalloc_SystemCommit(reinterpret_cast<void*>(span->start << kPageShift), static_cast<size_t>(span->length << kPageShift));
+    span->decommitted = false;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+    free_committed_pages_ += span->length;
+#endif
+  }
+  
   const int extra = static_cast<int>(span->length - n);
   ASSERT(extra >= 0);
   if (extra > 0) {
     Span* leftover = NewSpan(span->start + n, extra);
     leftover->free = 1;
+    leftover->decommitted = false;
     Event(leftover, 'S', extra);
     RecordSpan(leftover);
 
     // Place leftover span on appropriate free list
     SpanList* listpair = (static_cast<size_t>(extra) < kMaxPages) ? &free_[extra] : &large_;
-    Span* dst = released ? &listpair->returned : &listpair->normal;
+    Span* dst = &listpair->normal;
     DLL_Prepend(dst, leftover);
 
     span->length = n;
@@ -1278,6 +1829,18 @@ inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) {
   }
 }
 
+static ALWAYS_INLINE void mergeDecommittedStates(Span* destination, Span* other)
+{
+    if (destination->decommitted && !other->decommitted) {
+        TCMalloc_SystemRelease(reinterpret_cast<void*>(other->start << kPageShift),
+                               static_cast<size_t>(other->length << kPageShift));
+    } else if (other->decommitted && !destination->decommitted) {
+        TCMalloc_SystemRelease(reinterpret_cast<void*>(destination->start << kPageShift),
+                               static_cast<size_t>(destination->length << kPageShift));
+        destination->decommitted = true;
+    }
+}
+
 inline void TCMalloc_PageHeap::Delete(Span* span) {
   ASSERT(Check());
   ASSERT(!span->free);
@@ -1285,16 +1848,18 @@ inline void TCMalloc_PageHeap::Delete(Span* span) {
   ASSERT(GetDescriptor(span->start) == span);
   ASSERT(GetDescriptor(span->start + span->length - 1) == span);
   span->sizeclass = 0;
+#ifndef NO_TCMALLOC_SAMPLES
   span->sample = 0;
+#endif
 
   // Coalesce -- we guarantee that "p" != 0, so no bounds checking
   // necessary.  We do not bother resetting the stale pagemap
   // entries for the pieces we are merging together because we only
   // care about the pagemap entries for the boundaries.
-  //
-  // Note that the spans we merge into "span" may come out of
-  // a "returned" list.  For simplicity, we move these into the
-  // "normal" list of the appropriate size class.
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+  // Track the total size of the neighboring free spans that are committed.
+  Length neighboringCommittedSpansLength = 0;
+#endif
   const PageID p = span->start;
   const Length n = span->length;
   Span* prev = GetDescriptor(p-1);
@@ -1302,6 +1867,11 @@ inline void TCMalloc_PageHeap::Delete(Span* span) {
     // Merge preceding span into this span
     ASSERT(prev->start + prev->length == p);
     const Length len = prev->length;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+    if (!prev->decommitted)
+        neighboringCommittedSpansLength += len;
+#endif
+    mergeDecommittedStates(span, prev);
     DLL_Remove(prev);
     DeleteSpan(prev);
     span->start -= len;
@@ -1314,6 +1884,11 @@ inline void TCMalloc_PageHeap::Delete(Span* span) {
     // Merge next span into this span
     ASSERT(next->start == p+n);
     const Length len = next->length;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+    if (!next->decommitted)
+        neighboringCommittedSpansLength += len;
+#endif
+    mergeDecommittedStates(span, next);
     DLL_Remove(next);
     DeleteSpan(next);
     span->length += len;
@@ -1323,25 +1898,47 @@ inline void TCMalloc_PageHeap::Delete(Span* span) {
 
   Event(span, 'D', span->length);
   span->free = 1;
-  if (span->length < kMaxPages) {
-    DLL_Prepend(&free_[span->length].normal, span);
+  if (span->decommitted) {
+    if (span->length < kMaxPages)
+      DLL_Prepend(&free_[span->length].returned, span);
+    else
+      DLL_Prepend(&large_.returned, span);
   } else {
-    DLL_Prepend(&large_.normal, span);
+    if (span->length < kMaxPages)
+      DLL_Prepend(&free_[span->length].normal, span);
+    else
+      DLL_Prepend(&large_.normal, span);
   }
   free_pages_ += n;
 
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+  if (span->decommitted) {
+      // If the merged span is decommitted, that means we decommitted any neighboring spans that were
+      // committed.  Update the free committed pages count.
+      free_committed_pages_ -= neighboringCommittedSpansLength;
+      if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_)
+            min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+  } else {
+      // If the merged span remains committed, add the deleted span's size to the free committed pages count.
+      free_committed_pages_ += n;
+  }
+
+  // Make sure the scavenge thread becomes active if we have enough freed pages to release some back to the system.
+  signalScavenger();
+#else
   IncrementalScavenge(n);
+#endif
+
   ASSERT(Check());
 }
 
+#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
 void TCMalloc_PageHeap::IncrementalScavenge(Length n) {
   // Fast path; not yet time to release memory
   scavenge_counter_ -= n;
   if (scavenge_counter_ >= 0) return;  // Not yet time to scavenge
 
-  // If there is nothing to release, wait for so many pages before
-  // scavenging again.  With 4K pages, this comes to 16MB of memory.
-  static const size_t kDefaultReleaseDelay = 1 << 8;
+  static const size_t kDefaultReleaseDelay = 64;
 
   // Find index of free list to scavenge
   size_t index = scavenge_index_ + 1;
@@ -1354,9 +1951,10 @@ void TCMalloc_PageHeap::IncrementalScavenge(Length n) {
       DLL_Remove(s);
       TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
                              static_cast<size_t>(s->length << kPageShift));
+      s->decommitted = true;
       DLL_Prepend(&slist->returned, s);
 
-      scavenge_counter_ = std::max<size_t>(64UL, std::min<size_t>(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay)));
+      scavenge_counter_ = std::max<size_t>(16UL, std::min<size_t>(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay)));
 
       if (index == kMaxPages && !DLL_IsEmpty(&slist->normal))
         scavenge_index_ = index - 1;
@@ -1370,6 +1968,7 @@ void TCMalloc_PageHeap::IncrementalScavenge(Length n) {
   // Nothing to scavenge, delay for a while
   scavenge_counter_ = kDefaultReleaseDelay;
 }
+#endif
 
 void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) {
   // Associate span object with all interior pages as well
@@ -1382,6 +1981,21 @@ void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) {
     pagemap_.set(span->start+i, span);
   }
 }
+    
+#ifdef WTF_CHANGES
+size_t TCMalloc_PageHeap::ReturnedBytes() const {
+    size_t result = 0;
+    for (unsigned s = 0; s < kMaxPages; s++) {
+        const int r_length = DLL_Length(&free_[s].returned);
+        unsigned r_pages = s * r_length;
+        result += r_pages << kPageShift;
+    }
+    
+    for (Span* s = large_.returned.next; s != &large_.returned; s = s->next)
+        result += s->length << kPageShift;
+    return result;
+}
+#endif
 
 #ifndef WTF_CHANGES
 static double PagesToMB(uint64_t pages) {
@@ -1460,7 +2074,7 @@ bool TCMalloc_PageHeap::GrowHeap(Length n) {
     if (n < ask) {
       // Try growing just "n" pages
       ask = n;
-      ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);;
+      ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
     }
     if (ptr == NULL) return false;
   }
@@ -1503,27 +2117,28 @@ bool TCMalloc_PageHeap::GrowHeap(Length n) {
 bool TCMalloc_PageHeap::Check() {
   ASSERT(free_[0].normal.next == &free_[0].normal);
   ASSERT(free_[0].returned.next == &free_[0].returned);
-  CheckList(&large_.normal, kMaxPages, 1000000000);
-  CheckList(&large_.returned, kMaxPages, 1000000000);
+  CheckList(&large_.normal, kMaxPages, 1000000000, false);
+  CheckList(&large_.returned, kMaxPages, 1000000000, true);
   for (Length s = 1; s < kMaxPages; s++) {
-    CheckList(&free_[s].normal, s, s);
-    CheckList(&free_[s].returned, s, s);
+    CheckList(&free_[s].normal, s, s, false);
+    CheckList(&free_[s].returned, s, s, true);
   }
   return true;
 }
 
 #if ASSERT_DISABLED
-bool TCMalloc_PageHeap::CheckList(Span*, Length, Length) {
+bool TCMalloc_PageHeap::CheckList(Span*, Length, Length, bool) {
   return true;
 }
 #else
-bool TCMalloc_PageHeap::CheckList(Span* list, Length min_pages, Length max_pages) {
+bool TCMalloc_PageHeap::CheckList(Span* list, Length min_pages, Length max_pages, bool decommitted) {
   for (Span* s = list->next; s != list; s = s->next) {
     CHECK_CONDITION(s->free);
     CHECK_CONDITION(s->length >= min_pages);
     CHECK_CONDITION(s->length <= max_pages);
     CHECK_CONDITION(GetDescriptor(s->start) == s);
     CHECK_CONDITION(GetDescriptor(s->start+s->length-1) == s);
+    CHECK_CONDITION(s->decommitted == decommitted);
   }
   return true;
 }
@@ -1535,6 +2150,7 @@ static void ReleaseFreeList(Span* list, Span* returned) {
   while (!DLL_IsEmpty(list)) {
     Span* s = list->prev;
     DLL_Remove(s);
+    s->decommitted = true;
     DLL_Prepend(returned, s);
     TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
                            static_cast<size_t>(s->length << kPageShift));
@@ -1608,7 +2224,7 @@ class TCMalloc_ThreadCache_FreeList {
   template <class Finder, class Reader>
   void enumerateFreeObjects(Finder& finder, const Reader& reader)
   {
-      for (void* nextObject = list_; nextObject; nextObject = *reader(reinterpret_cast<void**>(nextObject)))
+      for (void* nextObject = list_; nextObject; nextObject = reader.nextEntryInLinkedList(reinterpret_cast<void**>(nextObject)))
           finder.visit(nextObject);
   }
 #endif
@@ -1621,7 +2237,7 @@ class TCMalloc_ThreadCache_FreeList {
 class TCMalloc_ThreadCache {
  private:
   typedef TCMalloc_ThreadCache_FreeList FreeList;
-#if COMPILER(MSVC)
+#if OS(WINDOWS)
   typedef DWORD ThreadIdentifier;
 #else
   typedef pthread_t ThreadIdentifier;
@@ -1655,10 +2271,10 @@ class TCMalloc_ThreadCache {
   // Total byte size in cache
   size_t Size() const { return size_; }
 
-  void* Allocate(size_t size);
+  ALWAYS_INLINE void* Allocate(size_t size);
   void Deallocate(void* ptr, size_t size_class);
 
-  void FetchFromCentralCache(size_t cl, size_t allocationSize);
+  ALWAYS_INLINE void FetchFromCentralCache(size_t cl, size_t allocationSize);
   void ReleaseToCentralCache(size_t cl, int N);
   void Scavenge();
   void Print() const;
@@ -1721,14 +2337,19 @@ class TCMalloc_Central_FreeList {
 
 #ifdef WTF_CHANGES
   template <class Finder, class Reader>
-  void enumerateFreeObjects(Finder& finder, const Reader& reader)
+  void enumerateFreeObjects(Finder& finder, const Reader& reader, TCMalloc_Central_FreeList* remoteCentralFreeList)
   {
     for (Span* span = &empty_; span && span != &empty_; span = (span->next ? reader(span->next) : 0))
       ASSERT(!span->objects);
 
     ASSERT(!nonempty_.objects);
-    for (Span* span = reader(nonempty_.next); span && span != &nonempty_; span = (span->next ? reader(span->next) : 0)) {
-      for (void* nextObject = span->objects; nextObject; nextObject = *reader(reinterpret_cast<void**>(nextObject)))
+    static const ptrdiff_t nonemptyOffset = reinterpret_cast<const char*>(&nonempty_) - reinterpret_cast<const char*>(this);
+
+    Span* remoteNonempty = reinterpret_cast<Span*>(reinterpret_cast<char*>(remoteCentralFreeList) + nonemptyOffset);
+    Span* remoteSpan = nonempty_.next;
+
+    for (Span* span = reader(remoteSpan); span && remoteSpan != remoteNonempty; remoteSpan = span->next, span = (span->next ? reader(span->next) : 0)) {
+      for (void* nextObject = span->objects; nextObject; nextObject = reader.nextEntryInLinkedList(reinterpret_cast<void**>(nextObject)))
         finder.visit(nextObject);
     }
   }
@@ -1754,12 +2375,12 @@ class TCMalloc_Central_FreeList {
   // REQUIRES: lock_ is held
   // Release an object to spans.
   // May temporarily release lock_.
-  void ReleaseToSpans(void* object);
+  ALWAYS_INLINE void ReleaseToSpans(void* object);
 
   // REQUIRES: lock_ is held
   // Populate cache by fetching from the page heap.
   // May temporarily release lock_.
-  void Populate();
+  ALWAYS_INLINE void Populate();
 
   // REQUIRES: lock is held.
   // Tries to make room for a TCEntry.  If the cache is full it will try to
@@ -1772,7 +2393,7 @@ class TCMalloc_Central_FreeList {
   // just iterates over the sizeclasses but does so without taking a lock.
   // Returns true on success.
   // May temporarily lock a "random" size class.
-  static bool EvictRandomSizeClass(size_t locked_size_class, bool force);
+  static ALWAYS_INLINE bool EvictRandomSizeClass(size_t locked_size_class, bool force);
 
   // REQUIRES: lock_ is *not* held.
   // Tries to shrink the Cache.  If force is true it will relase objects to
@@ -1822,13 +2443,7 @@ class TCMalloc_Central_FreeListPadded : public TCMalloc_Central_FreeList {
 static TCMalloc_Central_FreeListPadded central_cache[kNumClasses];
 
 // Page-level allocator
-static SpinLock pageheap_lock = SPINLOCK_INITIALIZER;
-
-#if PLATFORM(ARM)
-static void* pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(void*) - 1) / sizeof(void*)] __attribute__((aligned));
-#else
-static void* pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(void*) - 1) / sizeof(void*)];
-#endif
+static AllocAlignmentInteger pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(AllocAlignmentInteger) - 1) / sizeof(AllocAlignmentInteger)];
 static bool phinited = false;
 
 // Avoid extra level of indirection by making "pageheap" be just an alias
@@ -1846,6 +2461,59 @@ static inline TCMalloc_PageHeap* getPageHeap()
 
 #define pageheap getPageHeap()
 
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
+#if HAVE(DISPATCH_H) || OS(WINDOWS)
+
+void TCMalloc_PageHeap::periodicScavenge()
+{
+    SpinLockHolder h(&pageheap_lock);
+    pageheap->scavenge();
+
+    if (shouldScavenge()) {
+        rescheduleScavenger();
+        return;
+    }
+
+    suspendScavenger();
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger()
+{
+    ASSERT(pageheap_lock.IsHeld());
+    if (isScavengerSuspended() && shouldScavenge())
+        scheduleScavenger();
+}
+
+#else
+
+void TCMalloc_PageHeap::scavengerThread()
+{
+#if HAVE(PTHREAD_SETNAME_NP)
+  pthread_setname_np("JavaScriptCore: FastMalloc scavenger");
+#endif
+
+  while (1) {
+      if (!shouldScavenge()) {
+          pthread_mutex_lock(&m_scavengeMutex);
+          m_scavengeThreadActive = false;
+          // Block until there are enough free committed pages to release back to the system.
+          pthread_cond_wait(&m_scavengeCondition, &m_scavengeMutex);
+          m_scavengeThreadActive = true;
+          pthread_mutex_unlock(&m_scavengeMutex);
+      }
+      sleep(kScavengeDelayInSeconds);
+      {
+          SpinLockHolder h(&pageheap_lock);
+          pageheap->scavenge();
+      }
+  }
+}
+
+#endif
+
+#endif
+
 // If TLS is available, we also store a copy
 // of the per-thread object in a __thread variable
 // since __thread variables are faster to read
@@ -1862,17 +2530,29 @@ static __thread TCMalloc_ThreadCache *threadlocal_heap;
 // Therefore, we use TSD keys only after tsd_inited is set to true.
 // Until then, we use a slow path to get the heap object.
 static bool tsd_inited = false;
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+static const pthread_key_t heap_key = __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY0;
+#else
 static pthread_key_t heap_key;
-#if COMPILER(MSVC)
+#endif
+#if OS(WINDOWS)
 DWORD tlsIndex = TLS_OUT_OF_INDEXES;
 #endif
 
 static ALWAYS_INLINE void setThreadHeap(TCMalloc_ThreadCache* heap)
 {
-    // still do pthread_setspecific when using MSVC fast TLS to
-    // benefit from the delete callback.
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+    // Can't have two libraries both doing this in the same process,
+    // so check and make this crash right away.
+    if (pthread_getspecific(heap_key))
+        CRASH();
+#endif
+
+    // Still do pthread_setspecific even if there's an alternate form
+    // of thread-local storage in use, to benefit from the delete callback.
     pthread_setspecific(heap_key, heap);
-#if COMPILER(MSVC)
+
+#if OS(WINDOWS)
     TlsSetValue(tlsIndex, heap);
 #endif
 }
@@ -1933,7 +2613,7 @@ ALWAYS_INLINE void TCMalloc_Central_FreeList::ReleaseToSpans(void* object) {
   // The following check is expensive, so it is disabled by default
   if (false) {
     // Check that object does not occur in list
-    int got = 0;
+    unsigned got = 0;
     for (void* p = span->objects; p != NULL; p = *((void**) p)) {
       ASSERT(p != object);
       got++;
@@ -2099,6 +2779,7 @@ void* TCMalloc_Central_FreeList::FetchFromSpans() {
   Span* span = nonempty_.next;
 
   ASSERT(span->objects != NULL);
+  ASSERT_SPAN_COMMITTED(span);
   span->refcount++;
   void* result = span->objects;
   span->objects = *(reinterpret_cast<void**>(result));
@@ -2125,10 +2806,17 @@ ALWAYS_INLINE void TCMalloc_Central_FreeList::Populate() {
     if (span) pageheap->RegisterSizeClass(span, size_class_);
   }
   if (span == NULL) {
+#if HAVE(ERRNO_H)
     MESSAGE("allocation failed: %d\n", errno);
+#elif OS(WINDOWS)
+    MESSAGE("allocation failed: %d\n", ::GetLastError());
+#else
+    MESSAGE("allocation failed\n");
+#endif
     lock_.Lock();
     return;
   }
+  ASSERT_SPAN_COMMITTED(span);
   ASSERT(span->length == npages);
   // Cache sizeclass info eagerly.  Locking is not necessary.
   // (Instead of being eager, we could just replace any stale info
@@ -2147,7 +2835,7 @@ ALWAYS_INLINE void TCMalloc_Central_FreeList::Populate() {
   char* nptr;
   while ((nptr = ptr + size) <= limit) {
     *tail = ptr;
-    tail = reinterpret_cast<void**>(ptr);
+    tail = reinterpret_cast_ptr<void**>(ptr);
     ptr = nptr;
     num++;
   }
@@ -2351,7 +3039,7 @@ void TCMalloc_ThreadCache::InitModule() {
     }
     pageheap->init();
     phinited = 1;
-#if defined(WTF_CHANGES) && PLATFORM(DARWIN)
+#if defined(WTF_CHANGES) && OS(DARWIN)
     FastMallocZone::init();
 #endif
   }
@@ -2375,7 +3063,7 @@ inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetThreadHeap() {
     // __thread is faster, but only when the kernel supports it
   if (KernelSupportsTLS())
     return threadlocal_heap;
-#elif COMPILER(MSVC)
+#elif OS(WINDOWS)
     return static_cast<TCMalloc_ThreadCache*>(TlsGetValue(tlsIndex));
 #else
     return static_cast<TCMalloc_ThreadCache*>(pthread_getspecific(heap_key));
@@ -2404,13 +3092,17 @@ inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetCacheIfPresent() {
 
 void TCMalloc_ThreadCache::InitTSD() {
   ASSERT(!tsd_inited);
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+  pthread_key_init_np(heap_key, DestroyThreadCache);
+#else
   pthread_key_create(&heap_key, DestroyThreadCache);
-#if COMPILER(MSVC)
+#endif
+#if OS(WINDOWS)
   tlsIndex = TlsAlloc();
 #endif
   tsd_inited = true;
     
-#if !COMPILER(MSVC)
+#if !OS(WINDOWS)
   // We may have used a fake pthread_t for the main thread.  Fix it.
   pthread_t zero;
   memset(&zero, 0, sizeof(zero));
@@ -2421,7 +3113,7 @@ void TCMalloc_ThreadCache::InitTSD() {
   ASSERT(pageheap_lock.IsHeld());
 #endif
   for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) {
-#if COMPILER(MSVC)
+#if OS(WINDOWS)
     if (h->tid_ == 0) {
       h->tid_ = GetCurrentThreadId();
     }
@@ -2439,7 +3131,7 @@ TCMalloc_ThreadCache* TCMalloc_ThreadCache::CreateCacheIfNecessary() {
   {
     SpinLockHolder h(&pageheap_lock);
 
-#if COMPILER(MSVC)
+#if OS(WINDOWS)
     DWORD me;
     if (!tsd_inited) {
       me = 0;
@@ -2460,7 +3152,7 @@ TCMalloc_ThreadCache* TCMalloc_ThreadCache::CreateCacheIfNecessary() {
     // In that case, the heap for this thread has already been created
     // and added to the linked list.  So we search for that first.
     for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) {
-#if COMPILER(MSVC)
+#if OS(WINDOWS)
       if (h->tid_ == me) {
 #else
       if (pthread_equal(h->tid_, me)) {
@@ -2491,7 +3183,7 @@ void TCMalloc_ThreadCache::BecomeIdle() {
   if (heap->in_setspecific_) return;    // Do not disturb the active caller
 
   heap->in_setspecific_ = true;
-  pthread_setspecific(heap_key, NULL);
+  setThreadHeap(NULL);
 #ifdef HAVE_TLS
   // Also update the copy in __thread
   threadlocal_heap = NULL;
@@ -2905,11 +3597,15 @@ static inline void* CheckedMallocResult(void *result)
 }
 
 static inline void* SpanToMallocResult(Span *span) {
+  ASSERT_SPAN_COMMITTED(span);
   pageheap->CacheSizeClass(span->start, 0);
   return
       CheckedMallocResult(reinterpret_cast<void*>(span->start << kPageShift));
 }
 
+#ifdef WTF_CHANGES
+template <bool crashOnFailure>
+#endif
 static ALWAYS_INLINE void* do_malloc(size_t size) {
   void* ret = NULL;
 
@@ -2939,7 +3635,14 @@ static ALWAYS_INLINE void* do_malloc(size_t size) {
     // size-appropriate freelist, afer replenishing it if it's empty.
     ret = CheckedMallocResult(heap->Allocate(size));
   }
-  if (ret == NULL) errno = ENOMEM;
+  if (!ret) {
+#ifdef WTF_CHANGES
+    if (crashOnFailure) // This branch should be optimized out by the compiler.
+        CRASH();
+#else
+    errno = ENOMEM;
+#endif
+  }
   return ret;
 }
 
@@ -2956,7 +3659,9 @@ static ALWAYS_INLINE void do_free(void* ptr) {
     pageheap->CacheSizeClass(p, cl);
   }
   if (cl != 0) {
+#ifndef NO_TCMALLOC_SAMPLES
     ASSERT(!pageheap->GetDescriptor(p)->sample);
+#endif
     TCMalloc_ThreadCache* heap = TCMalloc_ThreadCache::GetCacheIfPresent();
     if (heap != NULL) {
       heap->Deallocate(ptr, cl);
@@ -2969,11 +3674,13 @@ static ALWAYS_INLINE void do_free(void* ptr) {
     SpinLockHolder h(&pageheap_lock);
     ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0);
     ASSERT(span != NULL && span->start == p);
+#ifndef NO_TCMALLOC_SAMPLES
     if (span->sample) {
       DLL_Remove(span);
       stacktrace_allocator.Delete(reinterpret_cast<StackTrace*>(span->objects));
       span->objects = NULL;
     }
+#endif
     pageheap->Delete(span);
   }
 }
@@ -3099,9 +3806,44 @@ static inline struct mallinfo do_mallinfo() {
 
 #ifndef WTF_CHANGES
 extern "C" 
+#else
+#define do_malloc do_malloc<crashOnFailure>
+
+template <bool crashOnFailure>
+ALWAYS_INLINE void* malloc(size_t);
+
+void* fastMalloc(size_t size)
+{
+    return malloc<true>(size);
+}
+
+TryMallocReturnValue tryFastMalloc(size_t size)
+{
+    return malloc<false>(size);
+}
+
+template <bool crashOnFailure>
+ALWAYS_INLINE
 #endif
 void* malloc(size_t size) {
-  void* result = do_malloc(size);
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    if (std::numeric_limits<size_t>::max() - Internal::ValidationBufferSize <= size)  // If overflow would occur...
+        return 0;
+    void* result = do_malloc(size + Internal::ValidationBufferSize);
+    if (!result)
+        return 0;
+
+    Internal::ValidationHeader* header = static_cast<Internal::ValidationHeader*>(result);
+    header->m_size = size;
+    header->m_type = Internal::AllocTypeMalloc;
+    header->m_prefix = static_cast<unsigned>(Internal::ValidationPrefix);
+    result = header + 1;
+    *Internal::fastMallocValidationSuffix(result) = Internal::ValidationSuffix;
+    fastMallocValidate(result);
+#else
+    void* result = do_malloc(size);
+#endif
+
 #ifndef WTF_CHANGES
   MallocHook::InvokeNewHook(result, size);
 #endif
@@ -3115,29 +3857,76 @@ void free(void* ptr) {
 #ifndef WTF_CHANGES
   MallocHook::InvokeDeleteHook(ptr);
 #endif
-  do_free(ptr);
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    if (!ptr)
+        return;
+
+    fastMallocValidate(ptr);
+    Internal::ValidationHeader* header = Internal::fastMallocValidationHeader(ptr);
+    memset(ptr, 0xCC, header->m_size);
+    do_free(header);
+#else
+    do_free(ptr);
+#endif
 }
 
 #ifndef WTF_CHANGES
 extern "C" 
+#else
+template <bool crashOnFailure>
+ALWAYS_INLINE void* calloc(size_t, size_t);
+
+void* fastCalloc(size_t n, size_t elem_size)
+{
+    void* result = calloc<true>(n, elem_size);
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    fastMallocValidate(result);
+#endif
+    return result;
+}
+
+TryMallocReturnValue tryFastCalloc(size_t n, size_t elem_size)
+{
+    void* result = calloc<false>(n, elem_size);
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    fastMallocValidate(result);
+#endif
+    return result;
+}
+
+template <bool crashOnFailure>
+ALWAYS_INLINE
 #endif
 void* calloc(size_t n, size_t elem_size) {
-  const size_t totalBytes = n * elem_size;
+  size_t totalBytes = n * elem_size;
     
   // Protect against overflow
   if (n > 1 && elem_size && (totalBytes / elem_size) != n)
     return 0;
-    
-  void* result = do_malloc(totalBytes);
-  if (result != NULL) {
+
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    void* result = malloc<crashOnFailure>(totalBytes);
+    if (!result)
+        return 0;
+
     memset(result, 0, totalBytes);
-  }
+    fastMallocValidate(result);
+#else
+    void* result = do_malloc(totalBytes);
+    if (result != NULL) {
+        memset(result, 0, totalBytes);
+    }
+#endif
+
 #ifndef WTF_CHANGES
   MallocHook::InvokeNewHook(result, totalBytes);
 #endif
   return result;
 }
 
+// Since cfree isn't used anywhere, we don't compile it in.
+#ifndef WTF_CHANGES
 #ifndef WTF_CHANGES
 extern "C" 
 #endif
@@ -3147,15 +3936,50 @@ void cfree(void* ptr) {
 #endif
   do_free(ptr);
 }
+#endif
 
 #ifndef WTF_CHANGES
 extern "C" 
+#else
+template <bool crashOnFailure>
+ALWAYS_INLINE void* realloc(void*, size_t);
+
+void* fastRealloc(void* old_ptr, size_t new_size)
+{
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    fastMallocValidate(old_ptr);
+#endif
+    void* result = realloc<true>(old_ptr, new_size);
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    fastMallocValidate(result);
+#endif
+    return result;
+}
+
+TryMallocReturnValue tryFastRealloc(void* old_ptr, size_t new_size)
+{
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    fastMallocValidate(old_ptr);
+#endif
+    void* result = realloc<false>(old_ptr, new_size);
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    fastMallocValidate(result);
+#endif
+    return result;
+}
+
+template <bool crashOnFailure>
+ALWAYS_INLINE
 #endif
 void* realloc(void* old_ptr, size_t new_size) {
   if (old_ptr == NULL) {
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    void* result = malloc<crashOnFailure>(new_size);
+#else
     void* result = do_malloc(new_size);
 #ifndef WTF_CHANGES
     MallocHook::InvokeNewHook(result, new_size);
+#endif
 #endif
     return result;
   }
@@ -3167,6 +3991,16 @@ void* realloc(void* old_ptr, size_t new_size) {
     return NULL;
   }
 
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    if (std::numeric_limits<size_t>::max() - Internal::ValidationBufferSize <= new_size)  // If overflow would occur...
+        return 0;
+    Internal::ValidationHeader* header = Internal::fastMallocValidationHeader(old_ptr);
+    fastMallocValidate(old_ptr);
+    old_ptr = header;
+    header->m_size = new_size;
+    new_size += Internal::ValidationBufferSize;
+#endif
+
   // Get the size of the old entry
   const PageID p = reinterpret_cast<uintptr_t>(old_ptr) >> kPageShift;
   size_t cl = pageheap->GetSizeClassIfCached(p);
@@ -3203,13 +4037,23 @@ void* realloc(void* old_ptr, size_t new_size) {
     // that we already know the sizeclass of old_ptr.  The benefit
     // would be small, so don't bother.
     do_free(old_ptr);
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    new_ptr = static_cast<Internal::ValidationHeader*>(new_ptr) + 1;
+    *Internal::fastMallocValidationSuffix(new_ptr) = Internal::ValidationSuffix;
+#endif
     return new_ptr;
   } else {
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    old_ptr = static_cast<Internal::ValidationHeader*>(old_ptr) + 1; // Set old_ptr back to the user pointer.
+    *Internal::fastMallocValidationSuffix(old_ptr) = Internal::ValidationSuffix;
+#endif
     return old_ptr;
   }
 }
 
-#ifndef WTF_CHANGES
+#ifdef WTF_CHANGES
+#undef do_malloc
+#else
 
 static SpinLock set_new_handler_lock = SPINLOCK_INITIALIZER;
 
@@ -3251,6 +4095,8 @@ static inline void* cpp_alloc(size_t size, bool nothrow) {
   }
 }
 
+#if ENABLE(GLOBAL_FASTMALLOC_NEW)
+
 void* operator new(size_t size) {
   void* p = cpp_alloc(size, false);
   // We keep this next instruction out of cpp_alloc for a reason: when
@@ -3305,6 +4151,8 @@ void operator delete[](void* p, const std::nothrow_t&) __THROW {
   do_free(p);
 }
 
+#endif
+
 extern "C" void* memalign(size_t align, size_t size) __THROW {
   void* result = do_memalign(align, size);
   MallocHook::InvokeNewHook(result, size);
@@ -3372,7 +4220,7 @@ extern "C" struct mallinfo mallinfo(void) {
 
 #if defined(__GLIBC__)
 extern "C" {
-# if defined(__GNUC__) && !defined(__MACH__) && defined(HAVE___ATTRIBUTE__)
+#if COMPILER(GCC) && !defined(__MACH__) && defined(HAVE___ATTRIBUTE__)
   // Potentially faster variants that use the gcc alias extension.
   // Mach-O (Darwin) does not support weak aliases, hence the __MACH__ check.
 # define ALIAS(x) __attribute__ ((weak, alias (x)))
@@ -3420,8 +4268,66 @@ void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride;
 
 #endif
 
-#if defined(WTF_CHANGES) && PLATFORM(DARWIN)
-#include <wtf/HashSet.h>
+#ifdef WTF_CHANGES
+void releaseFastMallocFreeMemory()
+{
+    // Flush free pages in the current thread cache back to the page heap.
+    // Low watermark mechanism in Scavenge() prevents full return on the first pass.
+    // The second pass flushes everything.
+    if (TCMalloc_ThreadCache* threadCache = TCMalloc_ThreadCache::GetCacheIfPresent()) {
+        threadCache->Scavenge();
+        threadCache->Scavenge();
+    }
+
+    SpinLockHolder h(&pageheap_lock);
+    pageheap->ReleaseFreePages();
+}
+    
+FastMallocStatistics fastMallocStatistics()
+{
+    FastMallocStatistics statistics;
+
+    SpinLockHolder lockHolder(&pageheap_lock);
+    statistics.reservedVMBytes = static_cast<size_t>(pageheap->SystemBytes());
+    statistics.committedVMBytes = statistics.reservedVMBytes - pageheap->ReturnedBytes();
+
+    statistics.freeListBytes = 0;
+    for (unsigned cl = 0; cl < kNumClasses; ++cl) {
+        const int length = central_cache[cl].length();
+        const int tc_length = central_cache[cl].tc_length();
+
+        statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length);
+    }
+    for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadCache = threadCache->next_)
+        statistics.freeListBytes += threadCache->Size();
+
+    return statistics;
+}
+
+size_t fastMallocSize(const void* ptr)
+{
+#if ENABLE(WTF_MALLOC_VALIDATION)
+    return Internal::fastMallocValidationHeader(const_cast<void*>(ptr))->m_size;
+#else
+    const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
+    Span* span = pageheap->GetDescriptorEnsureSafe(p);
+
+    if (!span || span->free)
+        return 0;
+
+    for (void* free = span->objects; free != NULL; free = *((void**) free)) {
+        if (ptr == free)
+            return 0;
+    }
+
+    if (size_t cl = span->sizeclass)
+        return ByteSizeForClass(cl);
+
+    return span->length << kPageShift;
+#endif
+}
+
+#if OS(DARWIN)
 
 class FreeObjectFinder {
     const RemoteMemoryReader& m_reader;
@@ -3432,6 +4338,7 @@ public:
 
     void visit(void* ptr) { m_freeObjects.add(ptr); }
     bool isFreeObject(void* ptr) const { return m_freeObjects.contains(ptr); }
+    bool isFreeObject(vm_address_t ptr) const { return isFreeObject(reinterpret_cast<void*>(ptr)); }
     size_t freeObjectCount() const { return m_freeObjects.size(); }
 
     void findFreeObjects(TCMalloc_ThreadCache* threadCache)
@@ -3440,10 +4347,10 @@ public:
             threadCache->enumerateFreeObjects(*this, m_reader);
     }
 
-    void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes)
+    void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes, TCMalloc_Central_FreeListPadded* remoteCentralFreeList)
     {
         for (unsigned i = 0; i < numSizes; i++)
-            centralFreeList[i].enumerateFreeObjects(*this, m_reader);
+            centralFreeList[i].enumerateFreeObjects(*this, m_reader, remoteCentralFreeList + i);
     }
 };
 
@@ -3463,12 +4370,15 @@ public:
             return 1;
 
         Span* span = m_reader(reinterpret_cast<Span*>(ptr));
+        if (!span)
+            return 1;
+
         if (span->free) {
             void* ptr = reinterpret_cast<void*>(span->start << kPageShift);
             m_freeObjectFinder.visit(ptr);
         } else if (span->sizeclass) {
             // Walk the free list of the small-object span, keeping track of each object seen
-            for (void* nextObject = span->objects; nextObject; nextObject = *m_reader(reinterpret_cast<void**>(nextObject)))
+            for (void* nextObject = span->objects; nextObject; nextObject = m_reader.nextEntryInLinkedList(reinterpret_cast<void**>(nextObject)))
                 m_freeObjectFinder.visit(nextObject);
         }
         return span->length;
@@ -3482,7 +4392,9 @@ class PageMapMemoryUsageRecorder {
     vm_range_recorder_t* m_recorder;
     const RemoteMemoryReader& m_reader;
     const FreeObjectFinder& m_freeObjectFinder;
-    mutable HashSet<void*> m_seenPointers;
+
+    HashSet<void*> m_seenPointers;
+    Vector<Span*> m_coalescedSpans;
 
 public:
     PageMapMemoryUsageRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader, const FreeObjectFinder& freeObjectFinder)
@@ -3494,51 +4406,133 @@ public:
         , m_freeObjectFinder(freeObjectFinder)
     { }
 
-    int visit(void* ptr) const
+    ~PageMapMemoryUsageRecorder()
+    {
+        ASSERT(!m_coalescedSpans.size());
+    }
+
+    void recordPendingRegions()
+    {
+        Span* lastSpan = m_coalescedSpans[m_coalescedSpans.size() - 1];
+        vm_range_t ptrRange = { m_coalescedSpans[0]->start << kPageShift, 0 };
+        ptrRange.size = (lastSpan->start << kPageShift) - ptrRange.address + (lastSpan->length * kPageSize);
+
+        // Mark the memory region the spans represent as a candidate for containing pointers
+        if (m_typeMask & MALLOC_PTR_REGION_RANGE_TYPE)
+            (*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1);
+
+        if (!(m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) {
+            m_coalescedSpans.clear();
+            return;
+        }
+
+        Vector<vm_range_t, 1024> allocatedPointers;
+        for (size_t i = 0; i < m_coalescedSpans.size(); ++i) {
+            Span *theSpan = m_coalescedSpans[i];
+            if (theSpan->free)
+                continue;
+
+            vm_address_t spanStartAddress = theSpan->start << kPageShift;
+            vm_size_t spanSizeInBytes = theSpan->length * kPageSize;
+
+            if (!theSpan->sizeclass) {
+                // If it's an allocated large object span, mark it as in use
+                if (!m_freeObjectFinder.isFreeObject(spanStartAddress))
+                    allocatedPointers.append((vm_range_t){spanStartAddress, spanSizeInBytes});
+            } else {
+                const size_t objectSize = ByteSizeForClass(theSpan->sizeclass);
+
+                // Mark each allocated small object within the span as in use
+                const vm_address_t endOfSpan = spanStartAddress + spanSizeInBytes;
+                for (vm_address_t object = spanStartAddress; object + objectSize <= endOfSpan; object += objectSize) {
+                    if (!m_freeObjectFinder.isFreeObject(object))
+                        allocatedPointers.append((vm_range_t){object, objectSize});
+                }
+            }
+        }
+
+        (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, allocatedPointers.data(), allocatedPointers.size());
+
+        m_coalescedSpans.clear();
+    }
+
+    int visit(void* ptr)
     {
         if (!ptr)
             return 1;
 
         Span* span = m_reader(reinterpret_cast<Span*>(ptr));
+        if (!span || !span->start)
+            return 1;
+
         if (m_seenPointers.contains(ptr))
             return span->length;
         m_seenPointers.add(ptr);
 
-        // Mark the memory used for the Span itself as an administrative region
-        vm_range_t ptrRange = { reinterpret_cast<vm_address_t>(ptr), sizeof(Span) };
-        if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE))
-            (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, &ptrRange, 1);
+        if (!m_coalescedSpans.size()) {
+            m_coalescedSpans.append(span);
+            return span->length;
+        }
 
-        ptrRange.address = span->start << kPageShift;
-        ptrRange.size = span->length * kPageSize;
+        Span* previousSpan = m_coalescedSpans[m_coalescedSpans.size() - 1];
+        vm_address_t previousSpanStartAddress = previousSpan->start << kPageShift;
+        vm_size_t previousSpanSizeInBytes = previousSpan->length * kPageSize;
 
-        // Mark the memory region the span represents as candidates for containing pointers
-        if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE))
-            (*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1);
+        // If the new span is adjacent to the previous span, do nothing for now.
+        vm_address_t spanStartAddress = span->start << kPageShift;
+        if (spanStartAddress == previousSpanStartAddress + previousSpanSizeInBytes) {
+            m_coalescedSpans.append(span);
+            return span->length;
+        }
 
-        if (!span->free && (m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) {
-            // If it's an allocated large object span, mark it as in use
-            if (span->sizeclass == 0 && !m_freeObjectFinder.isFreeObject(reinterpret_cast<void*>(ptrRange.address)))
-                (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &ptrRange, 1);
-            else if (span->sizeclass) {
-                const size_t byteSize = ByteSizeForClass(span->sizeclass);
-                unsigned totalObjects = (span->length << kPageShift) / byteSize;
-                ASSERT(span->refcount <= totalObjects);
-                char* ptr = reinterpret_cast<char*>(span->start << kPageShift);
+        // New span is not adjacent to previous span, so record the spans coalesced so far.
+        recordPendingRegions();
+        m_coalescedSpans.append(span);
 
-                // Mark each allocated small object within the span as in use
-                for (unsigned i = 0; i < totalObjects; i++) {
-                    char* thisObject = ptr + (i * byteSize);
-                    if (m_freeObjectFinder.isFreeObject(thisObject))
-                        continue;
+        return span->length;
+    }
+};
 
-                    vm_range_t objectRange = { reinterpret_cast<vm_address_t>(thisObject), byteSize };
-                    (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &objectRange, 1);
-                }
-            }
+class AdminRegionRecorder {
+    task_t m_task;
+    void* m_context;
+    unsigned m_typeMask;
+    vm_range_recorder_t* m_recorder;
+    const RemoteMemoryReader& m_reader;
+
+    Vector<vm_range_t, 1024> m_pendingRegions;
+
+public:
+    AdminRegionRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader)
+        : m_task(task)
+        , m_context(context)
+        , m_typeMask(typeMask)
+        , m_recorder(recorder)
+        , m_reader(reader)
+    { }
+
+    void recordRegion(vm_address_t ptr, size_t size)
+    {
+        if (m_typeMask & MALLOC_ADMIN_REGION_RANGE_TYPE)
+            m_pendingRegions.append((vm_range_t){ ptr, size });
+    }
+
+    void visit(void *ptr, size_t size)
+    {
+        recordRegion(reinterpret_cast<vm_address_t>(ptr), size);
+    }
+
+    void recordPendingRegions()
+    {
+        if (m_pendingRegions.size()) {
+            (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, m_pendingRegions.data(), m_pendingRegions.size());
+            m_pendingRegions.clear();
         }
+    }
 
-        return span->length;
+    ~AdminRegionRecorder()
+    {
+        ASSERT(!m_pendingRegions.size());
     }
 };
 
@@ -3557,14 +4551,26 @@ kern_return_t FastMallocZone::enumerate(task_t task, void* context, unsigned typ
 
     FreeObjectFinder finder(memoryReader);
     finder.findFreeObjects(threadHeaps);
-    finder.findFreeObjects(centralCaches, kNumClasses);
+    finder.findFreeObjects(centralCaches, kNumClasses, mzone->m_centralCaches);
 
     TCMalloc_PageHeap::PageMap* pageMap = &pageHeap->pagemap_;
     PageMapFreeObjectFinder pageMapFinder(memoryReader, finder);
-    pageMap->visit(pageMapFinder, memoryReader);
+    pageMap->visitValues(pageMapFinder, memoryReader);
 
     PageMapMemoryUsageRecorder usageRecorder(task, context, typeMask, recorder, memoryReader, finder);
-    pageMap->visit(usageRecorder, memoryReader);
+    pageMap->visitValues(usageRecorder, memoryReader);
+    usageRecorder.recordPendingRegions();
+
+    AdminRegionRecorder adminRegionRecorder(task, context, typeMask, recorder, memoryReader);
+    pageMap->visitAllocations(adminRegionRecorder, memoryReader);
+
+    PageHeapAllocator<Span>* spanAllocator = memoryReader(mzone->m_spanAllocator);
+    PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator = memoryReader(mzone->m_pageHeapAllocator);
+
+    spanAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader);
+    pageHeapAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader);
+
+    adminRegionRecorder.recordPendingRegions();
 
     return 0;
 }
@@ -3605,15 +4611,23 @@ void* FastMallocZone::zoneRealloc(malloc_zone_t*, void*, size_t)
 
 extern "C" {
 malloc_introspection_t jscore_fastmalloc_introspection = { &FastMallocZone::enumerate, &FastMallocZone::goodSize, &FastMallocZone::check, &FastMallocZone::print,
-    &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics };
+    &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics
+
+    , 0 // zone_locked will not be called on the zone unless it advertises itself as version five or higher.
+    , 0, 0, 0, 0 // These members will not be used unless the zone advertises itself as version seven or higher.
+
+    };
 }
 
-FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches)
+FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches, PageHeapAllocator<Span>* spanAllocator, PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator)
     : m_pageHeap(pageHeap)
     , m_threadHeaps(threadHeaps)
     , m_centralCaches(centralCaches)
+    , m_spanAllocator(spanAllocator)
+    , m_pageHeapAllocator(pageHeapAllocator)
 {
     memset(&m_zone, 0, sizeof(m_zone));
+    m_zone.version = 4;
     m_zone.zone_name = "JavaScriptCore FastMalloc";
     m_zone.size = &FastMallocZone::size;
     m_zone.malloc = &FastMallocZone::zoneMalloc;
@@ -3629,21 +4643,12 @@ FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache
 
 void FastMallocZone::init()
 {
-    static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache));
+    static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator);
 }
 
-extern "C" {
-void releaseFastMallocFreeMemory()
-{
-    SpinLockHolder h(&pageheap_lock);
-    pageheap->ReleaseFreePages();
-}
-}
+#endif // OS(DARWIN)
 
-#endif
-
-#if WTF_CHANGES
 } // namespace WTF
-#endif
+#endif // WTF_CHANGES
 
-#endif // USE_SYSTEM_MALLOC
+#endif // FORCE_SYSTEM_MALLOC