// Copyright (c) 2005, 2007, Google Inc.
// All rights reserved.
+// Copyright (C) 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
#include "FastMalloc.h"
#include "Assertions.h"
-#if USE(MULTIPLE_THREADS)
+#include <limits>
+#if ENABLE(JSC_MULTIPLE_THREADS)
#include <pthread.h>
#endif
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+#include <System/pthread_machdep.h>
+#endif
#ifndef NO_TCMALLOC_SAMPLES
#ifdef WTF_CHANGES
#endif
#endif
-#if !defined(USE_SYSTEM_MALLOC) && defined(NDEBUG)
+#if !(defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC) && defined(NDEBUG)
#define FORCE_SYSTEM_MALLOC 0
#else
#define FORCE_SYSTEM_MALLOC 1
#endif
+// Use a background thread to periodically scavenge memory to release back to the system
+// https://bugs.webkit.org/show_bug.cgi?id=27900: don't turn this on for Tiger until we have figured out why it caused a crash.
+#define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 0
+
#ifndef NDEBUG
namespace WTF {
-#if USE(MULTIPLE_THREADS)
+#if ENABLE(JSC_MULTIPLE_THREADS)
static pthread_key_t isForbiddenKey;
static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT;
static void initializeIsForbiddenKey()
pthread_key_create(&isForbiddenKey, 0);
}
+#if !ASSERT_DISABLED
static bool isForbidden()
{
pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey);
return !!pthread_getspecific(isForbiddenKey);
}
+#endif
void fastMallocForbid()
{
{
staticIsForbidden = false;
}
-#endif // USE(MULTIPLE_THREADS)
+#endif // ENABLE(JSC_MULTIPLE_THREADS)
} // namespace WTF
#endif // NDEBUG
#include <string.h>
namespace WTF {
-void *fastZeroedMalloc(size_t n)
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+
+namespace Internal {
+
+void fastMallocMatchFailed(void*)
{
- void *result = fastMalloc(n);
- if (!result)
- return 0;
- memset(result, 0, n);
-#ifndef WTF_CHANGES
- MallocHook::InvokeNewHook(result, n);
+ CRASH();
+}
+
+} // namespace Internal
+
#endif
+
+void* fastZeroedMalloc(size_t n)
+{
+ void* result = fastMalloc(n);
+ memset(result, 0, n);
return result;
}
+
+char* fastStrDup(const char* src)
+{
+ int len = strlen(src) + 1;
+ char* dup = static_cast<char*>(fastMalloc(len));
+
+ if (dup)
+ memcpy(dup, src, len);
+
+ return dup;
+}
+TryMallocReturnValue tryFastZeroedMalloc(size_t n)
+{
+ void* result;
+ if (!tryFastMalloc(n).getValue(result))
+ return 0;
+ memset(result, 0, n);
+ return result;
}
+} // namespace WTF
+
#if FORCE_SYSTEM_MALLOC
-#include <stdlib.h>
-#if !PLATFORM(WIN_OS)
- #include <pthread.h>
+#if PLATFORM(BREWMP)
+#include "brew/SystemMallocBrew.h"
+#endif
+
+#if OS(DARWIN)
+#include <malloc/malloc.h>
+#elif COMPILER(MSVC)
+#include <malloc.h>
#endif
namespace WTF {
-
-void *fastMalloc(size_t n)
+
+TryMallocReturnValue tryFastMalloc(size_t n)
{
ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= n) // If overflow would occur...
+ return 0;
+
+ void* result = malloc(n + sizeof(AllocAlignmentInteger));
+ if (!result)
+ return 0;
+
+ *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
+ result = static_cast<AllocAlignmentInteger*>(result) + 1;
+
+ return result;
+#else
return malloc(n);
+#endif
+}
+
+void* fastMalloc(size_t n)
+{
+ ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ TryMallocReturnValue returnValue = tryFastMalloc(n);
+ void* result;
+ returnValue.getValue(result);
+#else
+ void* result = malloc(n);
+#endif
+
+ if (!result) {
+#if PLATFORM(BREWMP)
+ // The behavior of malloc(0) is implementation defined.
+ // To make sure that fastMalloc never returns 0, retry with fastMalloc(1).
+ if (!n)
+ return fastMalloc(1);
+#endif
+ CRASH();
+ }
+
+ return result;
}
-void *fastCalloc(size_t n_elements, size_t element_size)
+TryMallocReturnValue tryFastCalloc(size_t n_elements, size_t element_size)
{
ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ size_t totalBytes = n_elements * element_size;
+ if (n_elements > 1 && element_size && (totalBytes / element_size) != n_elements || (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= totalBytes))
+ return 0;
+
+ totalBytes += sizeof(AllocAlignmentInteger);
+ void* result = malloc(totalBytes);
+ if (!result)
+ return 0;
+
+ memset(result, 0, totalBytes);
+ *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
+ result = static_cast<AllocAlignmentInteger*>(result) + 1;
+ return result;
+#else
return calloc(n_elements, element_size);
+#endif
+}
+
+void* fastCalloc(size_t n_elements, size_t element_size)
+{
+ ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ TryMallocReturnValue returnValue = tryFastCalloc(n_elements, element_size);
+ void* result;
+ returnValue.getValue(result);
+#else
+ void* result = calloc(n_elements, element_size);
+#endif
+
+ if (!result) {
+#if PLATFORM(BREWMP)
+ // If either n_elements or element_size is 0, the behavior of calloc is implementation defined.
+ // To make sure that fastCalloc never returns 0, retry with fastCalloc(1, 1).
+ if (!n_elements || !element_size)
+ return fastCalloc(1, 1);
+#endif
+ CRASH();
+ }
+
+ return result;
}
void fastFree(void* p)
{
ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (!p)
+ return;
+
+ AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p);
+ if (*header != Internal::AllocTypeMalloc)
+ Internal::fastMallocMatchFailed(p);
+ free(header);
+#else
free(p);
+#endif
}
-void *fastRealloc(void* p, size_t n)
+TryMallocReturnValue tryFastRealloc(void* p, size_t n)
{
ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (p) {
+ if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= n) // If overflow would occur...
+ return 0;
+ AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p);
+ if (*header != Internal::AllocTypeMalloc)
+ Internal::fastMallocMatchFailed(p);
+ void* result = realloc(header, n + sizeof(AllocAlignmentInteger));
+ if (!result)
+ return 0;
+
+ // This should not be needed because the value is already there:
+ // *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
+ result = static_cast<AllocAlignmentInteger*>(result) + 1;
+ return result;
+ } else {
+ return fastMalloc(n);
+ }
+#else
return realloc(p, n);
+#endif
}
-} // namespace WTF
+void* fastRealloc(void* p, size_t n)
+{
+ ASSERT(!isForbidden());
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ TryMallocReturnValue returnValue = tryFastRealloc(p, n);
+ void* result;
+ returnValue.getValue(result);
+#else
+ void* result = realloc(p, n);
+#endif
+
+ if (!result)
+ CRASH();
+ return result;
+}
-extern "C" {
void releaseFastMallocFreeMemory() { }
+
+FastMallocStatistics fastMallocStatistics()
+{
+ FastMallocStatistics statistics = { 0, 0, 0 };
+ return statistics;
}
-#if PLATFORM(DARWIN)
+size_t fastMallocSize(const void* p)
+{
+#if OS(DARWIN)
+ return malloc_size(p);
+#elif COMPILER(MSVC)
+ return _msize(const_cast<void*>(p));
+#else
+ return 1;
+#endif
+}
+
+} // namespace WTF
+
+#if OS(DARWIN)
// This symbol is present in the JavaScriptCore exports file even when FastMalloc is disabled.
// It will never be used in this case, so it's type and value are less interesting than its presence.
extern "C" const int jscore_fastmalloc_introspection = 0;
#endif
-#else
+#else // FORCE_SYSTEM_MALLOC
#if HAVE(STDINT_H)
#include <stdint.h>
#include "TCSystemAlloc.h"
#include <algorithm>
#include <errno.h>
-#include <new>
+#include <limits>
#include <pthread.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
+#if OS(UNIX)
+#include <unistd.h>
+#endif
#if COMPILER(MSVC)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif
-#if WTF_CHANGES
+#ifdef WTF_CHANGES
-#if PLATFORM(DARWIN)
+#if OS(DARWIN)
#include "MallocZoneSupport.h"
+#include <wtf/HashSet.h>
+#include <wtf/Vector.h>
#endif
+#if HAVE(DISPATCH_H)
+#include <dispatch/dispatch.h>
+#endif
+
#ifndef PRIuS
#define PRIuS "zu"
// call to the function on Mac OS X, and it's used in performance-critical code. So we
// use a function pointer. But that's not necessarily faster on other platforms, and we had
// problems with this technique on Windows, so we'll do this only on Mac OS X.
-#if PLATFORM(DARWIN)
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+#define pthread_getspecific(key) _pthread_getspecific_direct(key)
+#define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val))
+#else
+#if OS(DARWIN)
static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_getspecific;
#define pthread_getspecific(key) pthread_getspecific_function_pointer(key)
#endif
+#endif
#define DEFINE_VARIABLE(type, name, value, meaning) \
namespace FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead { \
#define MESSAGE LOG_ERROR
#define CHECK_CONDITION ASSERT
-#if PLATFORM(DARWIN)
+#if OS(DARWIN)
+struct Span;
+class TCMalloc_Central_FreeListPadded;
class TCMalloc_PageHeap;
class TCMalloc_ThreadCache;
-class TCMalloc_Central_FreeListPadded;
+template <typename T> class PageHeapAllocator;
class FastMallocZone {
public:
static void log(malloc_zone_t*, void*) { }
static void forceLock(malloc_zone_t*) { }
static void forceUnlock(malloc_zone_t*) { }
- static void statistics(malloc_zone_t*, malloc_statistics_t*) { }
+ static void statistics(malloc_zone_t*, malloc_statistics_t* stats) { memset(stats, 0, sizeof(malloc_statistics_t)); }
private:
- FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*);
+ FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*, PageHeapAllocator<Span>*, PageHeapAllocator<TCMalloc_ThreadCache>*);
static size_t size(malloc_zone_t*, const void*);
static void* zoneMalloc(malloc_zone_t*, size_t);
static void* zoneCalloc(malloc_zone_t*, size_t numItems, size_t size);
TCMalloc_PageHeap* m_pageHeap;
TCMalloc_ThreadCache** m_threadHeaps;
TCMalloc_Central_FreeListPadded* m_centralCaches;
+ PageHeapAllocator<Span>* m_spanAllocator;
+ PageHeapAllocator<TCMalloc_ThreadCache>* m_pageHeapAllocator;
};
#endif
static const size_t kPageMapBigAllocationThreshold = 128 << 20;
// Minimum number of pages to fetch from system at a time. Must be
-// significantly bigger than kBlockSize to amortize system-call
+// significantly bigger than kPageSize to amortize system-call
// overhead, and also to reduce external fragementation. Also, we
// should keep this value big because various incarnations of Linux
// have small limits on the number of mmap() regions per
// Lower and upper bounds on the per-thread cache sizes
static const size_t kMinThreadCacheSize = kMaxSize * 2;
-static const size_t kMaxThreadCacheSize = 2 << 20;
+static const size_t kMaxThreadCacheSize = 512 * 1024;
// Default bound on the total amount of thread caches
static const size_t kDefaultOverallThreadCacheSize = 16 << 20;
// Do some sanity checking on add_amount[]/shift_amount[]/class_array[]
if (ClassIndex(0) < 0) {
MESSAGE("Invalid class index %d for size 0\n", ClassIndex(0));
- abort();
+ CRASH();
}
if (static_cast<size_t>(ClassIndex(kMaxSize)) >= sizeof(class_array)) {
MESSAGE("Invalid class index %d for kMaxSize\n", ClassIndex(kMaxSize));
- abort();
+ CRASH();
}
// Compute the size classes we want to use
if (sc != kNumClasses) {
MESSAGE("wrong number of size classes: found %" PRIuS " instead of %d\n",
sc, int(kNumClasses));
- abort();
+ CRASH();
}
// Initialize the mapping arrays
const size_t sc = SizeClass(size);
if (sc == 0) {
MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size);
- abort();
+ CRASH();
}
if (sc > 1 && size <= class_to_size[sc-1]) {
MESSAGE("Allocating unnecessarily large class %" PRIuS " for %" PRIuS
"\n", sc, size);
- abort();
+ CRASH();
}
if (sc >= kNumClasses) {
MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size);
- abort();
+ CRASH();
}
const size_t s = class_to_size[sc];
if (size > s) {
MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc);
- abort();
+ CRASH();
}
if (s == 0) {
MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc);
- abort();
+ CRASH();
}
}
char* free_area_;
size_t free_avail_;
+ // Linked list of all regions allocated by this allocator
+ void* allocated_regions_;
+
// Free list of already carved objects
void* free_list_;
void Init() {
ASSERT(kAlignedSize <= kAllocIncrement);
inuse_ = 0;
+ allocated_regions_ = 0;
free_area_ = NULL;
free_avail_ = 0;
free_list_ = NULL;
} else {
if (free_avail_ < kAlignedSize) {
// Need more room
- free_area_ = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));
- if (free_area_ == NULL) abort();
- free_avail_ = kAllocIncrement;
+ char* new_allocation = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement));
+ if (!new_allocation)
+ CRASH();
+
+ *(void**)new_allocation = allocated_regions_;
+ allocated_regions_ = new_allocation;
+ free_area_ = new_allocation + kAlignedSize;
+ free_avail_ = kAllocIncrement - kAlignedSize;
}
result = free_area_;
free_area_ += kAlignedSize;
}
int inuse() const { return inuse_; }
+
+#if defined(WTF_CHANGES) && OS(DARWIN)
+ template <class Recorder>
+ void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader& reader)
+ {
+ vm_address_t adminAllocation = reinterpret_cast<vm_address_t>(allocated_regions_);
+ while (adminAllocation) {
+ recorder.recordRegion(adminAllocation, kAllocIncrement);
+ adminAllocation = *reader(reinterpret_cast<vm_address_t*>(adminAllocation));
+ }
+ }
+#endif
};
// -------------------------------------------------------------------------
Span* prev; // Used when in link list
void* objects; // Linked list of free objects
unsigned int free : 1; // Is the span free
+#ifndef NO_TCMALLOC_SAMPLES
unsigned int sample : 1; // Sampled object?
+#endif
unsigned int sizeclass : 8; // Size-class for small objects (or 0)
unsigned int refcount : 11; // Number of non-free objects
+ bool decommitted : 1;
#undef SPAN_HISTORY
#ifdef SPAN_HISTORY
#endif
};
+#define ASSERT_SPAN_COMMITTED(span) ASSERT(!span->decommitted)
+
#ifdef SPAN_HISTORY
void Event(Span* span, char op, int v = 0) {
span->history[span->nexthistory] = op;
return list->next == list;
}
-#ifndef WTF_CHANGES
static int DLL_Length(const Span* list) {
int result = 0;
for (Span* s = list->next; s != list; s = s->next) {
}
return result;
}
-#endif
#if 0 /* Not needed at the moment -- causes compiler warnings if not used */
static void DLL_Print(const char* label, const Span* list) {
typedef PackedCache<BITS, uint64_t> CacheType;
};
+#if defined(WTF_CHANGES)
+#if CPU(X86_64)
+// On all known X86-64 platforms, the upper 16 bits are always unused and therefore
+// can be excluded from the PageMap key.
+// See http://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details
+
+static const size_t kBitsUnusedOn64Bit = 16;
+#else
+static const size_t kBitsUnusedOn64Bit = 0;
+#endif
+
+// A three-level map for 64-bit machines
+template <> class MapSelector<64> {
+ public:
+ typedef TCMalloc_PageMap3<64 - kPageShift - kBitsUnusedOn64Bit> Type;
+ typedef PackedCache<64, uint64_t> CacheType;
+};
+#endif
+
// A two-level map for 32-bit machines
template <> class MapSelector<32> {
public:
- typedef TCMalloc_PageMap2<32-kPageShift> Type;
- typedef PackedCache<32-kPageShift, uint16_t> CacheType;
+ typedef TCMalloc_PageMap2<32 - kPageShift> Type;
+ typedef PackedCache<32 - kPageShift, uint16_t> CacheType;
};
// -------------------------------------------------------------------------
// contiguous runs of pages (called a "span").
// -------------------------------------------------------------------------
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+// The page heap maintains a free list for spans that are no longer in use by
+// the central cache or any thread caches. We use a background thread to
+// periodically scan the free list and release a percentage of it back to the OS.
+
+// If free_committed_pages_ exceeds kMinimumFreeCommittedPageCount, the
+// background thread:
+// - wakes up
+// - pauses for kScavengeDelayInSeconds
+// - returns to the OS a percentage of the memory that remained unused during
+// that pause (kScavengePercentage * min_free_committed_pages_since_last_scavenge_)
+// The goal of this strategy is to reduce memory pressure in a timely fashion
+// while avoiding thrashing the OS allocator.
+
+// Time delay before the page heap scavenger will consider returning pages to
+// the OS.
+static const int kScavengeDelayInSeconds = 2;
+
+// Approximate percentage of free committed pages to return to the OS in one
+// scavenge.
+static const float kScavengePercentage = .5f;
+
+// number of span lists to keep spans in when memory is returned.
+static const int kMinSpanListsWithSpans = 32;
+
+// Number of free committed pages that we want to keep around. The minimum number of pages used when there
+// is 1 span in each of the first kMinSpanListsWithSpans spanlists. Currently 528 pages.
+static const size_t kMinimumFreeCommittedPageCount = kMinSpanListsWithSpans * ((1.0f+kMinSpanListsWithSpans) / 2.0f);
+
+#endif
+
class TCMalloc_PageHeap {
public:
void init();
pagemap_.Ensure(p, 1);
return GetDescriptor(p);
}
+
+ size_t ReturnedBytes() const;
#endif
// Dump state to stderr
// Bytes allocated from system
uint64_t system_bytes_;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ // Number of pages kept in free lists that are still committed.
+ Length free_committed_pages_;
+
+ // Minimum number of free committed pages since last scavenge. (Can be 0 if
+ // we've committed new pages since the last scavenge.)
+ Length min_free_committed_pages_since_last_scavenge_;
+#endif
+
bool GrowHeap(Length n);
// REQUIRES span->length >= n
// span of exactly the specified length. Else, returns NULL.
Span* AllocLarge(Length n);
+#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
// Incrementally release some memory to the system.
// IncrementalScavenge(n) is called whenever n pages are freed.
void IncrementalScavenge(Length n);
+#endif
// Number of pages to deallocate before doing more scavenging
int64_t scavenge_counter_;
// Index of last free list we scavenged
size_t scavenge_index_;
-#if defined(WTF_CHANGES) && PLATFORM(DARWIN)
+#if defined(WTF_CHANGES) && OS(DARWIN)
friend class FastMallocZone;
#endif
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ void initializeScavenger();
+ ALWAYS_INLINE void signalScavenger();
+ void scavenge();
+ ALWAYS_INLINE bool shouldScavenge() const;
+
+#if !HAVE(DISPATCH_H)
+ static NO_RETURN_WITH_VALUE void* runScavengerThread(void*);
+ NO_RETURN void scavengerThread();
+
+ // Keeps track of whether the background thread is actively scavenging memory every kScavengeDelayInSeconds, or
+ // it's blocked waiting for more pages to be deleted.
+ bool m_scavengeThreadActive;
+
+ pthread_mutex_t m_scavengeMutex;
+ pthread_cond_t m_scavengeCondition;
+#else // !HAVE(DISPATCH_H)
+ void periodicScavenge();
+
+ dispatch_queue_t m_scavengeQueue;
+ dispatch_source_t m_scavengeTimer;
+ bool m_scavengingScheduled;
+#endif
+
+#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
};
void TCMalloc_PageHeap::init()
pagemap_cache_ = PageMapCache(0);
free_pages_ = 0;
system_bytes_ = 0;
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ free_committed_pages_ = 0;
+ min_free_committed_pages_since_last_scavenge_ = 0;
+#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
scavenge_counter_ = 0;
// Start scavenging at kMaxPages list
scavenge_index_ = kMaxPages-1;
DLL_Init(&free_[i].normal);
DLL_Init(&free_[i].returned);
}
+
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ initializeScavenger();
+#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
}
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
+#if !HAVE(DISPATCH_H)
+
+void TCMalloc_PageHeap::initializeScavenger()
+{
+ pthread_mutex_init(&m_scavengeMutex, 0);
+ pthread_cond_init(&m_scavengeCondition, 0);
+ m_scavengeThreadActive = true;
+ pthread_t thread;
+ pthread_create(&thread, 0, runScavengerThread, this);
+}
+
+void* TCMalloc_PageHeap::runScavengerThread(void* context)
+{
+ static_cast<TCMalloc_PageHeap*>(context)->scavengerThread();
+#if COMPILER(MSVC)
+ // Without this, Visual Studio will complain that this method does not return a value.
+ return 0;
+#endif
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger()
+{
+ if (!m_scavengeThreadActive && shouldScavenge())
+ pthread_cond_signal(&m_scavengeCondition);
+}
+
+#else // !HAVE(DISPATCH_H)
+
+void TCMalloc_PageHeap::initializeScavenger()
+{
+ m_scavengeQueue = dispatch_queue_create("com.apple.JavaScriptCore.FastMallocSavenger", NULL);
+ m_scavengeTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, m_scavengeQueue);
+ dispatch_time_t startTime = dispatch_time(DISPATCH_TIME_NOW, kScavengeDelayInSeconds * NSEC_PER_SEC);
+ dispatch_source_set_timer(m_scavengeTimer, startTime, kScavengeDelayInSeconds * NSEC_PER_SEC, 1000 * NSEC_PER_USEC);
+ dispatch_source_set_event_handler(m_scavengeTimer, ^{ periodicScavenge(); });
+ m_scavengingScheduled = false;
+}
+
+ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger()
+{
+ if (!m_scavengingScheduled && shouldScavenge()) {
+ m_scavengingScheduled = true;
+ dispatch_resume(m_scavengeTimer);
+ }
+}
+
+#endif
+
+void TCMalloc_PageHeap::scavenge()
+{
+ size_t pagesToRelease = min_free_committed_pages_since_last_scavenge_ * kScavengePercentage;
+ size_t targetPageCount = std::max<size_t>(kMinimumFreeCommittedPageCount, free_committed_pages_ - pagesToRelease);
+
+ while (free_committed_pages_ > targetPageCount) {
+ for (int i = kMaxPages; i > 0 && free_committed_pages_ >= targetPageCount; i--) {
+ SpanList* slist = (static_cast<size_t>(i) == kMaxPages) ? &large_ : &free_[i];
+ // If the span size is bigger than kMinSpanListsWithSpans pages return all the spans in the list, else return all but 1 span.
+ // Return only 50% of a spanlist at a time so spans of size 1 are not the only ones left.
+ size_t numSpansToReturn = (i > kMinSpanListsWithSpans) ? DLL_Length(&slist->normal) : static_cast<size_t>(.5 * DLL_Length(&slist->normal));
+ for (int j = 0; static_cast<size_t>(j) < numSpansToReturn && !DLL_IsEmpty(&slist->normal) && free_committed_pages_ > targetPageCount; j++) {
+ Span* s = slist->normal.prev;
+ DLL_Remove(s);
+ ASSERT(!s->decommitted);
+ if (!s->decommitted) {
+ TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
+ static_cast<size_t>(s->length << kPageShift));
+ ASSERT(free_committed_pages_ >= s->length);
+ free_committed_pages_ -= s->length;
+ s->decommitted = true;
+ }
+ DLL_Prepend(&slist->returned, s);
+ }
+ }
+ }
+
+ min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+}
+
+ALWAYS_INLINE bool TCMalloc_PageHeap::shouldScavenge() const
+{
+ return free_committed_pages_ > kMinimumFreeCommittedPageCount;
+}
+
+#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
inline Span* TCMalloc_PageHeap::New(Length n) {
ASSERT(Check());
ASSERT(n > 0);
Span* result = ll->next;
Carve(result, n, released);
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ // The newly allocated memory is from a span that's in the normal span list (already committed). Update the
+ // free committed pages count.
+ ASSERT(free_committed_pages_ >= n);
+ free_committed_pages_ -= n;
+ if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_)
+ min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
ASSERT(Check());
free_pages_ -= n;
return result;
}
Span* result = AllocLarge(n);
- if (result != NULL) return result;
+ if (result != NULL) {
+ ASSERT_SPAN_COMMITTED(result);
+ return result;
+ }
// Grow the heap and try again
if (!GrowHeap(n)) {
if (best != NULL) {
Carve(best, n, from_released);
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ // The newly allocated memory is from a span that's in the normal span list (already committed). Update the
+ // free committed pages count.
+ ASSERT(free_committed_pages_ >= n);
+ free_committed_pages_ -= n;
+ if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_)
+ min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
ASSERT(Check());
free_pages_ -= n;
return best;
span->free = 0;
Event(span, 'A', n);
+ if (released) {
+ // If the span chosen to carve from is decommited, commit the entire span at once to avoid committing spans 1 page at a time.
+ ASSERT(span->decommitted);
+ TCMalloc_SystemCommit(reinterpret_cast<void*>(span->start << kPageShift), static_cast<size_t>(span->length << kPageShift));
+ span->decommitted = false;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ free_committed_pages_ += span->length;
+#endif
+ }
+
const int extra = static_cast<int>(span->length - n);
ASSERT(extra >= 0);
if (extra > 0) {
Span* leftover = NewSpan(span->start + n, extra);
leftover->free = 1;
+ leftover->decommitted = false;
Event(leftover, 'S', extra);
RecordSpan(leftover);
// Place leftover span on appropriate free list
SpanList* listpair = (static_cast<size_t>(extra) < kMaxPages) ? &free_[extra] : &large_;
- Span* dst = released ? &listpair->returned : &listpair->normal;
+ Span* dst = &listpair->normal;
DLL_Prepend(dst, leftover);
span->length = n;
}
}
+static ALWAYS_INLINE void mergeDecommittedStates(Span* destination, Span* other)
+{
+ if (destination->decommitted && !other->decommitted) {
+ TCMalloc_SystemRelease(reinterpret_cast<void*>(other->start << kPageShift),
+ static_cast<size_t>(other->length << kPageShift));
+ } else if (other->decommitted && !destination->decommitted) {
+ TCMalloc_SystemRelease(reinterpret_cast<void*>(destination->start << kPageShift),
+ static_cast<size_t>(destination->length << kPageShift));
+ destination->decommitted = true;
+ }
+}
+
inline void TCMalloc_PageHeap::Delete(Span* span) {
ASSERT(Check());
ASSERT(!span->free);
ASSERT(GetDescriptor(span->start) == span);
ASSERT(GetDescriptor(span->start + span->length - 1) == span);
span->sizeclass = 0;
+#ifndef NO_TCMALLOC_SAMPLES
span->sample = 0;
+#endif
// Coalesce -- we guarantee that "p" != 0, so no bounds checking
// necessary. We do not bother resetting the stale pagemap
// entries for the pieces we are merging together because we only
// care about the pagemap entries for the boundaries.
- //
- // Note that the spans we merge into "span" may come out of
- // a "returned" list. For simplicity, we move these into the
- // "normal" list of the appropriate size class.
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ // Track the total size of the neighboring free spans that are committed.
+ Length neighboringCommittedSpansLength = 0;
+#endif
const PageID p = span->start;
const Length n = span->length;
Span* prev = GetDescriptor(p-1);
// Merge preceding span into this span
ASSERT(prev->start + prev->length == p);
const Length len = prev->length;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ if (!prev->decommitted)
+ neighboringCommittedSpansLength += len;
+#endif
+ mergeDecommittedStates(span, prev);
DLL_Remove(prev);
DeleteSpan(prev);
span->start -= len;
// Merge next span into this span
ASSERT(next->start == p+n);
const Length len = next->length;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ if (!next->decommitted)
+ neighboringCommittedSpansLength += len;
+#endif
+ mergeDecommittedStates(span, next);
DLL_Remove(next);
DeleteSpan(next);
span->length += len;
Event(span, 'D', span->length);
span->free = 1;
- if (span->length < kMaxPages) {
- DLL_Prepend(&free_[span->length].normal, span);
+ if (span->decommitted) {
+ if (span->length < kMaxPages)
+ DLL_Prepend(&free_[span->length].returned, span);
+ else
+ DLL_Prepend(&large_.returned, span);
} else {
- DLL_Prepend(&large_.normal, span);
+ if (span->length < kMaxPages)
+ DLL_Prepend(&free_[span->length].normal, span);
+ else
+ DLL_Prepend(&large_.normal, span);
}
free_pages_ += n;
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+ if (span->decommitted) {
+ // If the merged span is decommitted, that means we decommitted any neighboring spans that were
+ // committed. Update the free committed pages count.
+ free_committed_pages_ -= neighboringCommittedSpansLength;
+ if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_)
+ min_free_committed_pages_since_last_scavenge_ = free_committed_pages_;
+ } else {
+ // If the merged span remains committed, add the deleted span's size to the free committed pages count.
+ free_committed_pages_ += n;
+ }
+
+ // Make sure the scavenge thread becomes active if we have enough freed pages to release some back to the system.
+ signalScavenger();
+#else
IncrementalScavenge(n);
+#endif
+
ASSERT(Check());
}
+#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
void TCMalloc_PageHeap::IncrementalScavenge(Length n) {
// Fast path; not yet time to release memory
scavenge_counter_ -= n;
if (scavenge_counter_ >= 0) return; // Not yet time to scavenge
- // If there is nothing to release, wait for so many pages before
- // scavenging again. With 4K pages, this comes to 16MB of memory.
- static const size_t kDefaultReleaseDelay = 1 << 8;
+ static const size_t kDefaultReleaseDelay = 64;
// Find index of free list to scavenge
size_t index = scavenge_index_ + 1;
DLL_Remove(s);
TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
static_cast<size_t>(s->length << kPageShift));
+ s->decommitted = true;
DLL_Prepend(&slist->returned, s);
- scavenge_counter_ = std::max<size_t>(64UL, std::min<size_t>(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay)));
+ scavenge_counter_ = std::max<size_t>(16UL, std::min<size_t>(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay)));
if (index == kMaxPages && !DLL_IsEmpty(&slist->normal))
scavenge_index_ = index - 1;
// Nothing to scavenge, delay for a while
scavenge_counter_ = kDefaultReleaseDelay;
}
+#endif
void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) {
// Associate span object with all interior pages as well
pagemap_.set(span->start+i, span);
}
}
+
+#ifdef WTF_CHANGES
+size_t TCMalloc_PageHeap::ReturnedBytes() const {
+ size_t result = 0;
+ for (unsigned s = 0; s < kMaxPages; s++) {
+ const int r_length = DLL_Length(&free_[s].returned);
+ unsigned r_pages = s * r_length;
+ result += r_pages << kPageShift;
+ }
+
+ for (Span* s = large_.returned.next; s != &large_.returned; s = s->next)
+ result += s->length << kPageShift;
+ return result;
+}
+#endif
#ifndef WTF_CHANGES
static double PagesToMB(uint64_t pages) {
if (n < ask) {
// Try growing just "n" pages
ask = n;
- ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);;
+ ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
}
if (ptr == NULL) return false;
}
// Total byte size in cache
size_t Size() const { return size_; }
- void* Allocate(size_t size);
+ ALWAYS_INLINE void* Allocate(size_t size);
void Deallocate(void* ptr, size_t size_class);
- void FetchFromCentralCache(size_t cl, size_t allocationSize);
+ ALWAYS_INLINE void FetchFromCentralCache(size_t cl, size_t allocationSize);
void ReleaseToCentralCache(size_t cl, int N);
void Scavenge();
void Print() const;
#ifdef WTF_CHANGES
template <class Finder, class Reader>
- void enumerateFreeObjects(Finder& finder, const Reader& reader)
+ void enumerateFreeObjects(Finder& finder, const Reader& reader, TCMalloc_Central_FreeList* remoteCentralFreeList)
{
for (Span* span = &empty_; span && span != &empty_; span = (span->next ? reader(span->next) : 0))
ASSERT(!span->objects);
ASSERT(!nonempty_.objects);
- for (Span* span = reader(nonempty_.next); span && span != &nonempty_; span = (span->next ? reader(span->next) : 0)) {
+ static const ptrdiff_t nonemptyOffset = reinterpret_cast<const char*>(&nonempty_) - reinterpret_cast<const char*>(this);
+
+ Span* remoteNonempty = reinterpret_cast<Span*>(reinterpret_cast<char*>(remoteCentralFreeList) + nonemptyOffset);
+ Span* remoteSpan = nonempty_.next;
+
+ for (Span* span = reader(remoteSpan); span && remoteSpan != remoteNonempty; remoteSpan = span->next, span = (span->next ? reader(span->next) : 0)) {
for (void* nextObject = span->objects; nextObject; nextObject = *reader(reinterpret_cast<void**>(nextObject)))
finder.visit(nextObject);
}
// REQUIRES: lock_ is held
// Release an object to spans.
// May temporarily release lock_.
- void ReleaseToSpans(void* object);
+ ALWAYS_INLINE void ReleaseToSpans(void* object);
// REQUIRES: lock_ is held
// Populate cache by fetching from the page heap.
// May temporarily release lock_.
- void Populate();
+ ALWAYS_INLINE void Populate();
// REQUIRES: lock is held.
// Tries to make room for a TCEntry. If the cache is full it will try to
// just iterates over the sizeclasses but does so without taking a lock.
// Returns true on success.
// May temporarily lock a "random" size class.
- static bool EvictRandomSizeClass(size_t locked_size_class, bool force);
+ static ALWAYS_INLINE bool EvictRandomSizeClass(size_t locked_size_class, bool force);
// REQUIRES: lock_ is *not* held.
// Tries to shrink the Cache. If force is true it will relase objects to
#if PLATFORM(ARM)
static void* pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(void*) - 1) / sizeof(void*)] __attribute__((aligned));
#else
-static void* pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(void*) - 1) / sizeof(void*)];
+static AllocAlignmentInteger pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(AllocAlignmentInteger) - 1) / sizeof(AllocAlignmentInteger)];
#endif
static bool phinited = false;
#define pageheap getPageHeap()
+#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY
+
+#if !HAVE(DISPATCH_H)
+#if OS(WINDOWS)
+static void sleep(unsigned seconds)
+{
+ ::Sleep(seconds * 1000);
+}
+#endif
+
+void TCMalloc_PageHeap::scavengerThread()
+{
+#if HAVE(PTHREAD_SETNAME_NP)
+ pthread_setname_np("JavaScriptCore: FastMalloc scavenger");
+#endif
+
+ while (1) {
+ if (!shouldScavenge()) {
+ pthread_mutex_lock(&m_scavengeMutex);
+ m_scavengeThreadActive = false;
+ // Block until there are enough free committed pages to release back to the system.
+ pthread_cond_wait(&m_scavengeCondition, &m_scavengeMutex);
+ m_scavengeThreadActive = true;
+ pthread_mutex_unlock(&m_scavengeMutex);
+ }
+ sleep(kScavengeDelayInSeconds);
+ {
+ SpinLockHolder h(&pageheap_lock);
+ pageheap->scavenge();
+ }
+ }
+}
+
+#else
+
+void TCMalloc_PageHeap::periodicScavenge()
+{
+ {
+ SpinLockHolder h(&pageheap_lock);
+ pageheap->scavenge();
+ }
+
+ if (!shouldScavenge()) {
+ m_scavengingScheduled = false;
+ dispatch_suspend(m_scavengeTimer);
+ }
+}
+#endif // HAVE(DISPATCH_H)
+
+#endif
+
// If TLS is available, we also store a copy
// of the per-thread object in a __thread variable
// since __thread variables are faster to read
// Therefore, we use TSD keys only after tsd_inited is set to true.
// Until then, we use a slow path to get the heap object.
static bool tsd_inited = false;
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+static pthread_key_t heap_key = __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY0;
+#else
static pthread_key_t heap_key;
+#endif
#if COMPILER(MSVC)
DWORD tlsIndex = TLS_OUT_OF_INDEXES;
#endif
// The following check is expensive, so it is disabled by default
if (false) {
// Check that object does not occur in list
- int got = 0;
+ unsigned got = 0;
for (void* p = span->objects; p != NULL; p = *((void**) p)) {
ASSERT(p != object);
got++;
Span* span = nonempty_.next;
ASSERT(span->objects != NULL);
+ ASSERT_SPAN_COMMITTED(span);
span->refcount++;
void* result = span->objects;
span->objects = *(reinterpret_cast<void**>(result));
lock_.Lock();
return;
}
+ ASSERT_SPAN_COMMITTED(span);
ASSERT(span->length == npages);
// Cache sizeclass info eagerly. Locking is not necessary.
// (Instead of being eager, we could just replace any stale info
}
pageheap->init();
phinited = 1;
-#if defined(WTF_CHANGES) && PLATFORM(DARWIN)
+#if defined(WTF_CHANGES) && OS(DARWIN)
FastMallocZone::init();
#endif
}
void TCMalloc_ThreadCache::InitTSD() {
ASSERT(!tsd_inited);
+#if USE(PTHREAD_GETSPECIFIC_DIRECT)
+ pthread_key_init_np(heap_key, DestroyThreadCache);
+#else
pthread_key_create(&heap_key, DestroyThreadCache);
+#endif
#if COMPILER(MSVC)
tlsIndex = TlsAlloc();
#endif
}
static inline void* SpanToMallocResult(Span *span) {
+ ASSERT_SPAN_COMMITTED(span);
pageheap->CacheSizeClass(span->start, 0);
return
CheckedMallocResult(reinterpret_cast<void*>(span->start << kPageShift));
}
+#ifdef WTF_CHANGES
+template <bool crashOnFailure>
+#endif
static ALWAYS_INLINE void* do_malloc(size_t size) {
void* ret = NULL;
// size-appropriate freelist, afer replenishing it if it's empty.
ret = CheckedMallocResult(heap->Allocate(size));
}
- if (ret == NULL) errno = ENOMEM;
+ if (!ret) {
+#ifdef WTF_CHANGES
+ if (crashOnFailure) // This branch should be optimized out by the compiler.
+ CRASH();
+#else
+ errno = ENOMEM;
+#endif
+ }
return ret;
}
pageheap->CacheSizeClass(p, cl);
}
if (cl != 0) {
+#ifndef NO_TCMALLOC_SAMPLES
ASSERT(!pageheap->GetDescriptor(p)->sample);
+#endif
TCMalloc_ThreadCache* heap = TCMalloc_ThreadCache::GetCacheIfPresent();
if (heap != NULL) {
heap->Deallocate(ptr, cl);
SpinLockHolder h(&pageheap_lock);
ASSERT(reinterpret_cast<uintptr_t>(ptr) % kPageSize == 0);
ASSERT(span != NULL && span->start == p);
+#ifndef NO_TCMALLOC_SAMPLES
if (span->sample) {
DLL_Remove(span);
stacktrace_allocator.Delete(reinterpret_cast<StackTrace*>(span->objects));
span->objects = NULL;
}
+#endif
pageheap->Delete(span);
}
}
#ifndef WTF_CHANGES
extern "C"
+#else
+#define do_malloc do_malloc<crashOnFailure>
+
+template <bool crashOnFailure>
+ALWAYS_INLINE void* malloc(size_t);
+
+void* fastMalloc(size_t size)
+{
+ return malloc<true>(size);
+}
+
+TryMallocReturnValue tryFastMalloc(size_t size)
+{
+ return malloc<false>(size);
+}
+
+template <bool crashOnFailure>
+ALWAYS_INLINE
#endif
void* malloc(size_t size) {
- void* result = do_malloc(size);
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= size) // If overflow would occur...
+ return 0;
+ size += sizeof(AllocAlignmentInteger);
+ void* result = do_malloc(size);
+ if (!result)
+ return 0;
+
+ *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
+ result = static_cast<AllocAlignmentInteger*>(result) + 1;
+#else
+ void* result = do_malloc(size);
+#endif
+
#ifndef WTF_CHANGES
MallocHook::InvokeNewHook(result, size);
#endif
#ifndef WTF_CHANGES
MallocHook::InvokeDeleteHook(ptr);
#endif
- do_free(ptr);
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (!ptr)
+ return;
+
+ AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(ptr);
+ if (*header != Internal::AllocTypeMalloc)
+ Internal::fastMallocMatchFailed(ptr);
+ do_free(header);
+#else
+ do_free(ptr);
+#endif
}
#ifndef WTF_CHANGES
extern "C"
+#else
+template <bool crashOnFailure>
+ALWAYS_INLINE void* calloc(size_t, size_t);
+
+void* fastCalloc(size_t n, size_t elem_size)
+{
+ return calloc<true>(n, elem_size);
+}
+
+TryMallocReturnValue tryFastCalloc(size_t n, size_t elem_size)
+{
+ return calloc<false>(n, elem_size);
+}
+
+template <bool crashOnFailure>
+ALWAYS_INLINE
#endif
void* calloc(size_t n, size_t elem_size) {
- const size_t totalBytes = n * elem_size;
+ size_t totalBytes = n * elem_size;
// Protect against overflow
if (n > 1 && elem_size && (totalBytes / elem_size) != n)
return 0;
-
- void* result = do_malloc(totalBytes);
- if (result != NULL) {
+
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= totalBytes) // If overflow would occur...
+ return 0;
+
+ totalBytes += sizeof(AllocAlignmentInteger);
+ void* result = do_malloc(totalBytes);
+ if (!result)
+ return 0;
+
memset(result, 0, totalBytes);
- }
+ *static_cast<AllocAlignmentInteger*>(result) = Internal::AllocTypeMalloc;
+ result = static_cast<AllocAlignmentInteger*>(result) + 1;
+#else
+ void* result = do_malloc(totalBytes);
+ if (result != NULL) {
+ memset(result, 0, totalBytes);
+ }
+#endif
+
#ifndef WTF_CHANGES
MallocHook::InvokeNewHook(result, totalBytes);
#endif
return result;
}
+// Since cfree isn't used anywhere, we don't compile it in.
+#ifndef WTF_CHANGES
#ifndef WTF_CHANGES
extern "C"
#endif
#endif
do_free(ptr);
}
+#endif
#ifndef WTF_CHANGES
extern "C"
+#else
+template <bool crashOnFailure>
+ALWAYS_INLINE void* realloc(void*, size_t);
+
+void* fastRealloc(void* old_ptr, size_t new_size)
+{
+ return realloc<true>(old_ptr, new_size);
+}
+
+TryMallocReturnValue tryFastRealloc(void* old_ptr, size_t new_size)
+{
+ return realloc<false>(old_ptr, new_size);
+}
+
+template <bool crashOnFailure>
+ALWAYS_INLINE
#endif
void* realloc(void* old_ptr, size_t new_size) {
if (old_ptr == NULL) {
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ void* result = malloc(new_size);
+#else
void* result = do_malloc(new_size);
#ifndef WTF_CHANGES
MallocHook::InvokeNewHook(result, new_size);
+#endif
#endif
return result;
}
return NULL;
}
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ if (std::numeric_limits<size_t>::max() - sizeof(AllocAlignmentInteger) <= new_size) // If overflow would occur...
+ return 0;
+ new_size += sizeof(AllocAlignmentInteger);
+ AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(old_ptr);
+ if (*header != Internal::AllocTypeMalloc)
+ Internal::fastMallocMatchFailed(old_ptr);
+ old_ptr = header;
+#endif
+
// Get the size of the old entry
const PageID p = reinterpret_cast<uintptr_t>(old_ptr) >> kPageShift;
size_t cl = pageheap->GetSizeClassIfCached(p);
// that we already know the sizeclass of old_ptr. The benefit
// would be small, so don't bother.
do_free(old_ptr);
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ new_ptr = static_cast<AllocAlignmentInteger*>(new_ptr) + 1;
+#endif
return new_ptr;
} else {
+#if ENABLE(FAST_MALLOC_MATCH_VALIDATION)
+ old_ptr = static_cast<AllocAlignmentInteger*>(old_ptr) + 1; // Set old_ptr back to the user pointer.
+#endif
return old_ptr;
}
}
-#ifndef WTF_CHANGES
+#ifdef WTF_CHANGES
+#undef do_malloc
+#else
static SpinLock set_new_handler_lock = SPINLOCK_INITIALIZER;
}
}
+#if ENABLE(GLOBAL_FASTMALLOC_NEW)
+
void* operator new(size_t size) {
void* p = cpp_alloc(size, false);
// We keep this next instruction out of cpp_alloc for a reason: when
do_free(p);
}
+#endif
+
extern "C" void* memalign(size_t align, size_t size) __THROW {
void* result = do_memalign(align, size);
MallocHook::InvokeNewHook(result, size);
#if defined(__GLIBC__)
extern "C" {
-# if defined(__GNUC__) && !defined(__MACH__) && defined(HAVE___ATTRIBUTE__)
+#if COMPILER(GCC) && !defined(__MACH__) && defined(HAVE___ATTRIBUTE__)
// Potentially faster variants that use the gcc alias extension.
// Mach-O (Darwin) does not support weak aliases, hence the __MACH__ check.
# define ALIAS(x) __attribute__ ((weak, alias (x)))
#endif
-#if defined(WTF_CHANGES) && PLATFORM(DARWIN)
-#include <wtf/HashSet.h>
+#ifdef WTF_CHANGES
+void releaseFastMallocFreeMemory()
+{
+ // Flush free pages in the current thread cache back to the page heap.
+ // Low watermark mechanism in Scavenge() prevents full return on the first pass.
+ // The second pass flushes everything.
+ if (TCMalloc_ThreadCache* threadCache = TCMalloc_ThreadCache::GetCacheIfPresent()) {
+ threadCache->Scavenge();
+ threadCache->Scavenge();
+ }
+
+ SpinLockHolder h(&pageheap_lock);
+ pageheap->ReleaseFreePages();
+}
+
+FastMallocStatistics fastMallocStatistics()
+{
+ FastMallocStatistics statistics;
+
+ SpinLockHolder lockHolder(&pageheap_lock);
+ statistics.reservedVMBytes = static_cast<size_t>(pageheap->SystemBytes());
+ statistics.committedVMBytes = statistics.reservedVMBytes - pageheap->ReturnedBytes();
+
+ statistics.freeListBytes = 0;
+ for (unsigned cl = 0; cl < kNumClasses; ++cl) {
+ const int length = central_cache[cl].length();
+ const int tc_length = central_cache[cl].tc_length();
+
+ statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length);
+ }
+ for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadCache = threadCache->next_)
+ statistics.freeListBytes += threadCache->Size();
+
+ return statistics;
+}
+
+size_t fastMallocSize(const void* ptr)
+{
+ const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
+ Span* span = pageheap->GetDescriptorEnsureSafe(p);
+
+ if (!span || span->free)
+ return 0;
+
+ for (void* free = span->objects; free != NULL; free = *((void**) free)) {
+ if (ptr == free)
+ return 0;
+ }
+
+ if (size_t cl = span->sizeclass)
+ return ByteSizeForClass(cl);
+
+ return span->length << kPageShift;
+}
+
+#if OS(DARWIN)
class FreeObjectFinder {
const RemoteMemoryReader& m_reader;
void visit(void* ptr) { m_freeObjects.add(ptr); }
bool isFreeObject(void* ptr) const { return m_freeObjects.contains(ptr); }
+ bool isFreeObject(vm_address_t ptr) const { return isFreeObject(reinterpret_cast<void*>(ptr)); }
size_t freeObjectCount() const { return m_freeObjects.size(); }
void findFreeObjects(TCMalloc_ThreadCache* threadCache)
threadCache->enumerateFreeObjects(*this, m_reader);
}
- void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes)
+ void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes, TCMalloc_Central_FreeListPadded* remoteCentralFreeList)
{
for (unsigned i = 0; i < numSizes; i++)
- centralFreeList[i].enumerateFreeObjects(*this, m_reader);
+ centralFreeList[i].enumerateFreeObjects(*this, m_reader, remoteCentralFreeList + i);
}
};
vm_range_recorder_t* m_recorder;
const RemoteMemoryReader& m_reader;
const FreeObjectFinder& m_freeObjectFinder;
- mutable HashSet<void*> m_seenPointers;
+
+ HashSet<void*> m_seenPointers;
+ Vector<Span*> m_coalescedSpans;
public:
PageMapMemoryUsageRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader, const FreeObjectFinder& freeObjectFinder)
, m_freeObjectFinder(freeObjectFinder)
{ }
- int visit(void* ptr) const
+ ~PageMapMemoryUsageRecorder()
+ {
+ ASSERT(!m_coalescedSpans.size());
+ }
+
+ void recordPendingRegions()
+ {
+ Span* lastSpan = m_coalescedSpans[m_coalescedSpans.size() - 1];
+ vm_range_t ptrRange = { m_coalescedSpans[0]->start << kPageShift, 0 };
+ ptrRange.size = (lastSpan->start << kPageShift) - ptrRange.address + (lastSpan->length * kPageSize);
+
+ // Mark the memory region the spans represent as a candidate for containing pointers
+ if (m_typeMask & MALLOC_PTR_REGION_RANGE_TYPE)
+ (*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1);
+
+ if (!(m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) {
+ m_coalescedSpans.clear();
+ return;
+ }
+
+ Vector<vm_range_t, 1024> allocatedPointers;
+ for (size_t i = 0; i < m_coalescedSpans.size(); ++i) {
+ Span *theSpan = m_coalescedSpans[i];
+ if (theSpan->free)
+ continue;
+
+ vm_address_t spanStartAddress = theSpan->start << kPageShift;
+ vm_size_t spanSizeInBytes = theSpan->length * kPageSize;
+
+ if (!theSpan->sizeclass) {
+ // If it's an allocated large object span, mark it as in use
+ if (!m_freeObjectFinder.isFreeObject(spanStartAddress))
+ allocatedPointers.append((vm_range_t){spanStartAddress, spanSizeInBytes});
+ } else {
+ const size_t objectSize = ByteSizeForClass(theSpan->sizeclass);
+
+ // Mark each allocated small object within the span as in use
+ const vm_address_t endOfSpan = spanStartAddress + spanSizeInBytes;
+ for (vm_address_t object = spanStartAddress; object + objectSize <= endOfSpan; object += objectSize) {
+ if (!m_freeObjectFinder.isFreeObject(object))
+ allocatedPointers.append((vm_range_t){object, objectSize});
+ }
+ }
+ }
+
+ (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, allocatedPointers.data(), allocatedPointers.size());
+
+ m_coalescedSpans.clear();
+ }
+
+ int visit(void* ptr)
{
if (!ptr)
return 1;
Span* span = m_reader(reinterpret_cast<Span*>(ptr));
+ if (!span->start)
+ return 1;
+
if (m_seenPointers.contains(ptr))
return span->length;
m_seenPointers.add(ptr);
- // Mark the memory used for the Span itself as an administrative region
- vm_range_t ptrRange = { reinterpret_cast<vm_address_t>(ptr), sizeof(Span) };
- if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE))
- (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, &ptrRange, 1);
+ if (!m_coalescedSpans.size()) {
+ m_coalescedSpans.append(span);
+ return span->length;
+ }
- ptrRange.address = span->start << kPageShift;
- ptrRange.size = span->length * kPageSize;
+ Span* previousSpan = m_coalescedSpans[m_coalescedSpans.size() - 1];
+ vm_address_t previousSpanStartAddress = previousSpan->start << kPageShift;
+ vm_size_t previousSpanSizeInBytes = previousSpan->length * kPageSize;
- // Mark the memory region the span represents as candidates for containing pointers
- if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE))
- (*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1);
+ // If the new span is adjacent to the previous span, do nothing for now.
+ vm_address_t spanStartAddress = span->start << kPageShift;
+ if (spanStartAddress == previousSpanStartAddress + previousSpanSizeInBytes) {
+ m_coalescedSpans.append(span);
+ return span->length;
+ }
- if (!span->free && (m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) {
- // If it's an allocated large object span, mark it as in use
- if (span->sizeclass == 0 && !m_freeObjectFinder.isFreeObject(reinterpret_cast<void*>(ptrRange.address)))
- (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &ptrRange, 1);
- else if (span->sizeclass) {
- const size_t byteSize = ByteSizeForClass(span->sizeclass);
- unsigned totalObjects = (span->length << kPageShift) / byteSize;
- ASSERT(span->refcount <= totalObjects);
- char* ptr = reinterpret_cast<char*>(span->start << kPageShift);
+ // New span is not adjacent to previous span, so record the spans coalesced so far.
+ recordPendingRegions();
+ m_coalescedSpans.append(span);
- // Mark each allocated small object within the span as in use
- for (unsigned i = 0; i < totalObjects; i++) {
- char* thisObject = ptr + (i * byteSize);
- if (m_freeObjectFinder.isFreeObject(thisObject))
- continue;
+ return span->length;
+ }
+};
- vm_range_t objectRange = { reinterpret_cast<vm_address_t>(thisObject), byteSize };
- (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &objectRange, 1);
- }
- }
+class AdminRegionRecorder {
+ task_t m_task;
+ void* m_context;
+ unsigned m_typeMask;
+ vm_range_recorder_t* m_recorder;
+ const RemoteMemoryReader& m_reader;
+
+ Vector<vm_range_t, 1024> m_pendingRegions;
+
+public:
+ AdminRegionRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader)
+ : m_task(task)
+ , m_context(context)
+ , m_typeMask(typeMask)
+ , m_recorder(recorder)
+ , m_reader(reader)
+ { }
+
+ void recordRegion(vm_address_t ptr, size_t size)
+ {
+ if (m_typeMask & MALLOC_ADMIN_REGION_RANGE_TYPE)
+ m_pendingRegions.append((vm_range_t){ ptr, size });
+ }
+
+ void visit(void *ptr, size_t size)
+ {
+ recordRegion(reinterpret_cast<vm_address_t>(ptr), size);
+ }
+
+ void recordPendingRegions()
+ {
+ if (m_pendingRegions.size()) {
+ (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, m_pendingRegions.data(), m_pendingRegions.size());
+ m_pendingRegions.clear();
}
+ }
- return span->length;
+ ~AdminRegionRecorder()
+ {
+ ASSERT(!m_pendingRegions.size());
}
};
FreeObjectFinder finder(memoryReader);
finder.findFreeObjects(threadHeaps);
- finder.findFreeObjects(centralCaches, kNumClasses);
+ finder.findFreeObjects(centralCaches, kNumClasses, mzone->m_centralCaches);
TCMalloc_PageHeap::PageMap* pageMap = &pageHeap->pagemap_;
PageMapFreeObjectFinder pageMapFinder(memoryReader, finder);
- pageMap->visit(pageMapFinder, memoryReader);
+ pageMap->visitValues(pageMapFinder, memoryReader);
PageMapMemoryUsageRecorder usageRecorder(task, context, typeMask, recorder, memoryReader, finder);
- pageMap->visit(usageRecorder, memoryReader);
+ pageMap->visitValues(usageRecorder, memoryReader);
+ usageRecorder.recordPendingRegions();
+
+ AdminRegionRecorder adminRegionRecorder(task, context, typeMask, recorder, memoryReader);
+ pageMap->visitAllocations(adminRegionRecorder, memoryReader);
+
+ PageHeapAllocator<Span>* spanAllocator = memoryReader(mzone->m_spanAllocator);
+ PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator = memoryReader(mzone->m_pageHeapAllocator);
+
+ spanAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader);
+ pageHeapAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader);
+
+ adminRegionRecorder.recordPendingRegions();
return 0;
}
extern "C" {
malloc_introspection_t jscore_fastmalloc_introspection = { &FastMallocZone::enumerate, &FastMallocZone::goodSize, &FastMallocZone::check, &FastMallocZone::print,
- &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics };
+ &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics
+
+ , 0 // zone_locked will not be called on the zone unless it advertises itself as version five or higher.
+
+ };
}
-FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches)
+FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches, PageHeapAllocator<Span>* spanAllocator, PageHeapAllocator<TCMalloc_ThreadCache>* pageHeapAllocator)
: m_pageHeap(pageHeap)
, m_threadHeaps(threadHeaps)
, m_centralCaches(centralCaches)
+ , m_spanAllocator(spanAllocator)
+ , m_pageHeapAllocator(pageHeapAllocator)
{
memset(&m_zone, 0, sizeof(m_zone));
+ m_zone.version = 4;
m_zone.zone_name = "JavaScriptCore FastMalloc";
m_zone.size = &FastMallocZone::size;
m_zone.malloc = &FastMallocZone::zoneMalloc;
void FastMallocZone::init()
{
- static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache));
+ static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Central_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator);
}
-extern "C" {
-void releaseFastMallocFreeMemory()
-{
- SpinLockHolder h(&pageheap_lock);
- pageheap->ReleaseFreePages();
-}
-}
+#endif // OS(DARWIN)
-#endif
-
-#if WTF_CHANGES
} // namespace WTF
-#endif
+#endif // WTF_CHANGES
-#endif // USE_SYSTEM_MALLOC
+#endif // FORCE_SYSTEM_MALLOC
projects / apple / javascriptcore.git / blobdiff