X-Git-Url: https://git.saurik.com/apple/javascriptcore.git/blobdiff_plain/8537cb5c39a1e4bc9c6727383b9ac29b72019dfa..4e4e5a6f2694187498445a6ac6f1634ce8141119:/wtf/FastMalloc.cpp diff --git a/wtf/FastMalloc.cpp b/wtf/FastMalloc.cpp index f5f54f5..e3b25bd 100644 --- a/wtf/FastMalloc.cpp +++ b/wtf/FastMalloc.cpp @@ -1,6 +1,6 @@ // Copyright (c) 2005, 2007, Google Inc. // All rights reserved. -// Copyright (C) 2005, 2006, 2007, 2008 Apple Inc. All rights reserved. +// Copyright (C) 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -78,9 +78,13 @@ #include "FastMalloc.h" #include "Assertions.h" -#if USE(MULTIPLE_THREADS) +#include +#if ENABLE(JSC_MULTIPLE_THREADS) #include #endif +#if USE(PTHREAD_GETSPECIFIC_DIRECT) +#include +#endif #ifndef NO_TCMALLOC_SAMPLES #ifdef WTF_CHANGES @@ -88,16 +92,20 @@ #endif #endif -#if !defined(USE_SYSTEM_MALLOC) && defined(NDEBUG) +#if !(defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC) && defined(NDEBUG) #define FORCE_SYSTEM_MALLOC 0 #else #define FORCE_SYSTEM_MALLOC 1 #endif +// Use a background thread to periodically scavenge memory to release back to the system +// https://bugs.webkit.org/show_bug.cgi?id=27900: don't turn this on for Tiger until we have figured out why it caused a crash. +#define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 0 + #ifndef NDEBUG namespace WTF { -#if USE(MULTIPLE_THREADS) +#if ENABLE(JSC_MULTIPLE_THREADS) static pthread_key_t isForbiddenKey; static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT; static void initializeIsForbiddenKey() @@ -105,11 +113,13 @@ static void initializeIsForbiddenKey() pthread_key_create(&isForbiddenKey, 0); } +#if !ASSERT_DISABLED static bool isForbidden() { pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); return !!pthread_getspecific(isForbiddenKey); } +#endif void fastMallocForbid() { @@ -140,7 +150,7 @@ void fastMallocAllow() { staticIsForbidden = false; } -#endif // USE(MULTIPLE_THREADS) +#endif // ENABLE(JSC_MULTIPLE_THREADS) } // namespace WTF #endif // NDEBUG @@ -148,58 +158,240 @@ void fastMallocAllow() #include namespace WTF { -void *fastZeroedMalloc(size_t n) + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + +namespace Internal { + +void fastMallocMatchFailed(void*) { - void *result = fastMalloc(n); - if (!result) - return 0; - memset(result, 0, n); -#ifndef WTF_CHANGES - MallocHook::InvokeNewHook(result, n); + CRASH(); +} + +} // namespace Internal + #endif + +void* fastZeroedMalloc(size_t n) +{ + void* result = fastMalloc(n); + memset(result, 0, n); return result; } + +char* fastStrDup(const char* src) +{ + int len = strlen(src) + 1; + char* dup = static_cast(fastMalloc(len)); + + if (dup) + memcpy(dup, src, len); + + return dup; +} +TryMallocReturnValue tryFastZeroedMalloc(size_t n) +{ + void* result; + if (!tryFastMalloc(n).getValue(result)) + return 0; + memset(result, 0, n); + return result; } +} // namespace WTF + #if FORCE_SYSTEM_MALLOC -#include -#if !PLATFORM(WIN_OS) - #include +#if PLATFORM(BREWMP) +#include "brew/SystemMallocBrew.h" +#endif + +#if OS(DARWIN) +#include +#elif COMPILER(MSVC) +#include #endif namespace WTF { - -void *fastMalloc(size_t n) + +TryMallocReturnValue tryFastMalloc(size_t n) { ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (std::numeric_limits::max() - sizeof(AllocAlignmentInteger) <= n) // If overflow would occur... + return 0; + + void* result = malloc(n + sizeof(AllocAlignmentInteger)); + if (!result) + return 0; + + *static_cast(result) = Internal::AllocTypeMalloc; + result = static_cast(result) + 1; + + return result; +#else return malloc(n); +#endif +} + +void* fastMalloc(size_t n) +{ + ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + TryMallocReturnValue returnValue = tryFastMalloc(n); + void* result; + returnValue.getValue(result); +#else + void* result = malloc(n); +#endif + + if (!result) { +#if PLATFORM(BREWMP) + // The behavior of malloc(0) is implementation defined. + // To make sure that fastMalloc never returns 0, retry with fastMalloc(1). + if (!n) + return fastMalloc(1); +#endif + CRASH(); + } + + return result; } -void *fastCalloc(size_t n_elements, size_t element_size) +TryMallocReturnValue tryFastCalloc(size_t n_elements, size_t element_size) { ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + size_t totalBytes = n_elements * element_size; + if (n_elements > 1 && element_size && (totalBytes / element_size) != n_elements || (std::numeric_limits::max() - sizeof(AllocAlignmentInteger) <= totalBytes)) + return 0; + + totalBytes += sizeof(AllocAlignmentInteger); + void* result = malloc(totalBytes); + if (!result) + return 0; + + memset(result, 0, totalBytes); + *static_cast(result) = Internal::AllocTypeMalloc; + result = static_cast(result) + 1; + return result; +#else return calloc(n_elements, element_size); +#endif +} + +void* fastCalloc(size_t n_elements, size_t element_size) +{ + ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + TryMallocReturnValue returnValue = tryFastCalloc(n_elements, element_size); + void* result; + returnValue.getValue(result); +#else + void* result = calloc(n_elements, element_size); +#endif + + if (!result) { +#if PLATFORM(BREWMP) + // If either n_elements or element_size is 0, the behavior of calloc is implementation defined. + // To make sure that fastCalloc never returns 0, retry with fastCalloc(1, 1). + if (!n_elements || !element_size) + return fastCalloc(1, 1); +#endif + CRASH(); + } + + return result; } void fastFree(void* p) { ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (!p) + return; + + AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p); + if (*header != Internal::AllocTypeMalloc) + Internal::fastMallocMatchFailed(p); + free(header); +#else free(p); +#endif } -void *fastRealloc(void* p, size_t n) +TryMallocReturnValue tryFastRealloc(void* p, size_t n) { ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (p) { + if (std::numeric_limits::max() - sizeof(AllocAlignmentInteger) <= n) // If overflow would occur... + return 0; + AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(p); + if (*header != Internal::AllocTypeMalloc) + Internal::fastMallocMatchFailed(p); + void* result = realloc(header, n + sizeof(AllocAlignmentInteger)); + if (!result) + return 0; + + // This should not be needed because the value is already there: + // *static_cast(result) = Internal::AllocTypeMalloc; + result = static_cast(result) + 1; + return result; + } else { + return fastMalloc(n); + } +#else return realloc(p, n); +#endif +} + +void* fastRealloc(void* p, size_t n) +{ + ASSERT(!isForbidden()); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + TryMallocReturnValue returnValue = tryFastRealloc(p, n); + void* result; + returnValue.getValue(result); +#else + void* result = realloc(p, n); +#endif + + if (!result) + CRASH(); + return result; } void releaseFastMallocFreeMemory() { } + +FastMallocStatistics fastMallocStatistics() +{ + FastMallocStatistics statistics = { 0, 0, 0 }; + return statistics; +} + +size_t fastMallocSize(const void* p) +{ +#if OS(DARWIN) + return malloc_size(p); +#elif COMPILER(MSVC) + return _msize(const_cast(p)); +#else + return 1; +#endif +} } // namespace WTF -#if PLATFORM(DARWIN) +#if OS(DARWIN) // This symbol is present in the JavaScriptCore exports file even when FastMalloc is disabled. // It will never be used in this case, so it's type and value are less interesting than its presence. extern "C" const int jscore_fastmalloc_introspection = 0; @@ -223,11 +415,14 @@ extern "C" const int jscore_fastmalloc_introspection = 0; #include "TCSystemAlloc.h" #include #include -#include +#include #include #include #include #include +#if OS(UNIX) +#include +#endif #if COMPILER(MSVC) #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -235,12 +430,18 @@ extern "C" const int jscore_fastmalloc_introspection = 0; #include #endif -#if WTF_CHANGES +#ifdef WTF_CHANGES -#if PLATFORM(DARWIN) +#if OS(DARWIN) #include "MallocZoneSupport.h" +#include +#include +#endif +#if HAVE(DISPATCH_H) +#include #endif + #ifndef PRIuS #define PRIuS "zu" #endif @@ -249,10 +450,15 @@ extern "C" const int jscore_fastmalloc_introspection = 0; // call to the function on Mac OS X, and it's used in performance-critical code. So we // use a function pointer. But that's not necessarily faster on other platforms, and we had // problems with this technique on Windows, so we'll do this only on Mac OS X. -#if PLATFORM(DARWIN) +#if USE(PTHREAD_GETSPECIFIC_DIRECT) +#define pthread_getspecific(key) _pthread_getspecific_direct(key) +#define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val)) +#else +#if OS(DARWIN) static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_getspecific; #define pthread_getspecific(key) pthread_getspecific_function_pointer(key) #endif +#endif #define DEFINE_VARIABLE(type, name, value, meaning) \ namespace FLAG__namespace_do_not_use_directly_use_DECLARE_##type##_instead { \ @@ -277,10 +483,12 @@ namespace WTF { #define MESSAGE LOG_ERROR #define CHECK_CONDITION ASSERT -#if PLATFORM(DARWIN) +#if OS(DARWIN) +struct Span; +class TCMalloc_Central_FreeListPadded; class TCMalloc_PageHeap; class TCMalloc_ThreadCache; -class TCMalloc_Central_FreeListPadded; +template class PageHeapAllocator; class FastMallocZone { public: @@ -293,10 +501,10 @@ public: static void log(malloc_zone_t*, void*) { } static void forceLock(malloc_zone_t*) { } static void forceUnlock(malloc_zone_t*) { } - static void statistics(malloc_zone_t*, malloc_statistics_t*) { } + static void statistics(malloc_zone_t*, malloc_statistics_t* stats) { memset(stats, 0, sizeof(malloc_statistics_t)); } private: - FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*); + FastMallocZone(TCMalloc_PageHeap*, TCMalloc_ThreadCache**, TCMalloc_Central_FreeListPadded*, PageHeapAllocator*, PageHeapAllocator*); static size_t size(malloc_zone_t*, const void*); static void* zoneMalloc(malloc_zone_t*, size_t); static void* zoneCalloc(malloc_zone_t*, size_t numItems, size_t size); @@ -309,6 +517,8 @@ private: TCMalloc_PageHeap* m_pageHeap; TCMalloc_ThreadCache** m_threadHeaps; TCMalloc_Central_FreeListPadded* m_centralCaches; + PageHeapAllocator* m_spanAllocator; + PageHeapAllocator* m_pageHeapAllocator; }; #endif @@ -389,7 +599,7 @@ static const size_t kNumClasses = 68; static const size_t kPageMapBigAllocationThreshold = 128 << 20; // Minimum number of pages to fetch from system at a time. Must be -// significantly bigger than kBlockSize to amortize system-call +// significantly bigger than kPageSize to amortize system-call // overhead, and also to reduce external fragementation. Also, we // should keep this value big because various incarnations of Linux // have small limits on the number of mmap() regions per @@ -412,7 +622,7 @@ static const int kMaxFreeListLength = 256; // Lower and upper bounds on the per-thread cache sizes static const size_t kMinThreadCacheSize = kMaxSize * 2; -static const size_t kMaxThreadCacheSize = 2 << 20; +static const size_t kMaxThreadCacheSize = 512 * 1024; // Default bound on the total amount of thread caches static const size_t kDefaultOverallThreadCacheSize = 16 << 20; @@ -627,11 +837,11 @@ static void InitSizeClasses() { // Do some sanity checking on add_amount[]/shift_amount[]/class_array[] if (ClassIndex(0) < 0) { MESSAGE("Invalid class index %d for size 0\n", ClassIndex(0)); - abort(); + CRASH(); } if (static_cast(ClassIndex(kMaxSize)) >= sizeof(class_array)) { MESSAGE("Invalid class index %d for kMaxSize\n", ClassIndex(kMaxSize)); - abort(); + CRASH(); } // Compute the size classes we want to use @@ -683,7 +893,7 @@ static void InitSizeClasses() { if (sc != kNumClasses) { MESSAGE("wrong number of size classes: found %" PRIuS " instead of %d\n", sc, int(kNumClasses)); - abort(); + CRASH(); } // Initialize the mapping arrays @@ -701,25 +911,25 @@ static void InitSizeClasses() { const size_t sc = SizeClass(size); if (sc == 0) { MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size); - abort(); + CRASH(); } if (sc > 1 && size <= class_to_size[sc-1]) { MESSAGE("Allocating unnecessarily large class %" PRIuS " for %" PRIuS "\n", sc, size); - abort(); + CRASH(); } if (sc >= kNumClasses) { MESSAGE("Bad size class %" PRIuS " for %" PRIuS "\n", sc, size); - abort(); + CRASH(); } const size_t s = class_to_size[sc]; if (size > s) { MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc); - abort(); + CRASH(); } if (s == 0) { MESSAGE("Bad size %" PRIuS " for %" PRIuS " (sc = %" PRIuS ")\n", s, size, sc); - abort(); + CRASH(); } } @@ -777,6 +987,9 @@ class PageHeapAllocator { char* free_area_; size_t free_avail_; + // Linked list of all regions allocated by this allocator + void* allocated_regions_; + // Free list of already carved objects void* free_list_; @@ -787,6 +1000,7 @@ class PageHeapAllocator { void Init() { ASSERT(kAlignedSize <= kAllocIncrement); inuse_ = 0; + allocated_regions_ = 0; free_area_ = NULL; free_avail_ = 0; free_list_ = NULL; @@ -801,9 +1015,14 @@ class PageHeapAllocator { } else { if (free_avail_ < kAlignedSize) { // Need more room - free_area_ = reinterpret_cast(MetaDataAlloc(kAllocIncrement)); - if (free_area_ == NULL) abort(); - free_avail_ = kAllocIncrement; + char* new_allocation = reinterpret_cast(MetaDataAlloc(kAllocIncrement)); + if (!new_allocation) + CRASH(); + + *(void**)new_allocation = allocated_regions_; + allocated_regions_ = new_allocation; + free_area_ = new_allocation + kAlignedSize; + free_avail_ = kAllocIncrement - kAlignedSize; } result = free_area_; free_area_ += kAlignedSize; @@ -820,6 +1039,18 @@ class PageHeapAllocator { } int inuse() const { return inuse_; } + +#if defined(WTF_CHANGES) && OS(DARWIN) + template + void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader& reader) + { + vm_address_t adminAllocation = reinterpret_cast(allocated_regions_); + while (adminAllocation) { + recorder.recordRegion(adminAllocation, kAllocIncrement); + adminAllocation = *reader(reinterpret_cast(adminAllocation)); + } + } +#endif }; // ------------------------------------------------------------------------- @@ -862,9 +1093,12 @@ struct Span { Span* prev; // Used when in link list void* objects; // Linked list of free objects unsigned int free : 1; // Is the span free +#ifndef NO_TCMALLOC_SAMPLES unsigned int sample : 1; // Sampled object? +#endif unsigned int sizeclass : 8; // Size-class for small objects (or 0) unsigned int refcount : 11; // Number of non-free objects + bool decommitted : 1; #undef SPAN_HISTORY #ifdef SPAN_HISTORY @@ -875,6 +1109,8 @@ struct Span { #endif }; +#define ASSERT_SPAN_COMMITTED(span) ASSERT(!span->decommitted) + #ifdef SPAN_HISTORY void Event(Span* span, char op, int v = 0) { span->history[span->nexthistory] = op; @@ -927,7 +1163,6 @@ static ALWAYS_INLINE bool DLL_IsEmpty(const Span* list) { return list->next == list; } -#ifndef WTF_CHANGES static int DLL_Length(const Span* list) { int result = 0; for (Span* s = list->next; s != list; s = s->next) { @@ -935,7 +1170,6 @@ static int DLL_Length(const Span* list) { } return result; } -#endif #if 0 /* Not needed at the moment -- causes compiler warnings if not used */ static void DLL_Print(const char* label, const Span* list) { @@ -987,11 +1221,30 @@ template class MapSelector { typedef PackedCache CacheType; }; +#if defined(WTF_CHANGES) +#if CPU(X86_64) +// On all known X86-64 platforms, the upper 16 bits are always unused and therefore +// can be excluded from the PageMap key. +// See http://en.wikipedia.org/wiki/X86-64#Virtual_address_space_details + +static const size_t kBitsUnusedOn64Bit = 16; +#else +static const size_t kBitsUnusedOn64Bit = 0; +#endif + +// A three-level map for 64-bit machines +template <> class MapSelector<64> { + public: + typedef TCMalloc_PageMap3<64 - kPageShift - kBitsUnusedOn64Bit> Type; + typedef PackedCache<64, uint64_t> CacheType; +}; +#endif + // A two-level map for 32-bit machines template <> class MapSelector<32> { public: - typedef TCMalloc_PageMap2<32-kPageShift> Type; - typedef PackedCache<32-kPageShift, uint16_t> CacheType; + typedef TCMalloc_PageMap2<32 - kPageShift> Type; + typedef PackedCache<32 - kPageShift, uint16_t> CacheType; }; // ------------------------------------------------------------------------- @@ -1002,6 +1255,37 @@ template <> class MapSelector<32> { // contiguous runs of pages (called a "span"). // ------------------------------------------------------------------------- +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY +// The page heap maintains a free list for spans that are no longer in use by +// the central cache or any thread caches. We use a background thread to +// periodically scan the free list and release a percentage of it back to the OS. + +// If free_committed_pages_ exceeds kMinimumFreeCommittedPageCount, the +// background thread: +// - wakes up +// - pauses for kScavengeDelayInSeconds +// - returns to the OS a percentage of the memory that remained unused during +// that pause (kScavengePercentage * min_free_committed_pages_since_last_scavenge_) +// The goal of this strategy is to reduce memory pressure in a timely fashion +// while avoiding thrashing the OS allocator. + +// Time delay before the page heap scavenger will consider returning pages to +// the OS. +static const int kScavengeDelayInSeconds = 2; + +// Approximate percentage of free committed pages to return to the OS in one +// scavenge. +static const float kScavengePercentage = .5f; + +// number of span lists to keep spans in when memory is returned. +static const int kMinSpanListsWithSpans = 32; + +// Number of free committed pages that we want to keep around. The minimum number of pages used when there +// is 1 span in each of the first kMinSpanListsWithSpans spanlists. Currently 528 pages. +static const size_t kMinimumFreeCommittedPageCount = kMinSpanListsWithSpans * ((1.0f+kMinSpanListsWithSpans) / 2.0f); + +#endif + class TCMalloc_PageHeap { public: void init(); @@ -1041,6 +1325,8 @@ class TCMalloc_PageHeap { pagemap_.Ensure(p, 1); return GetDescriptor(p); } + + size_t ReturnedBytes() const; #endif // Dump state to stderr @@ -1099,6 +1385,15 @@ class TCMalloc_PageHeap { // Bytes allocated from system uint64_t system_bytes_; +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + // Number of pages kept in free lists that are still committed. + Length free_committed_pages_; + + // Minimum number of free committed pages since last scavenge. (Can be 0 if + // we've committed new pages since the last scavenge.) + Length min_free_committed_pages_since_last_scavenge_; +#endif + bool GrowHeap(Length n); // REQUIRES span->length >= n @@ -1121,9 +1416,11 @@ class TCMalloc_PageHeap { // span of exactly the specified length. Else, returns NULL. Span* AllocLarge(Length n); +#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY // Incrementally release some memory to the system. // IncrementalScavenge(n) is called whenever n pages are freed. void IncrementalScavenge(Length n); +#endif // Number of pages to deallocate before doing more scavenging int64_t scavenge_counter_; @@ -1131,9 +1428,35 @@ class TCMalloc_PageHeap { // Index of last free list we scavenged size_t scavenge_index_; -#if defined(WTF_CHANGES) && PLATFORM(DARWIN) +#if defined(WTF_CHANGES) && OS(DARWIN) friend class FastMallocZone; #endif + +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + void initializeScavenger(); + ALWAYS_INLINE void signalScavenger(); + void scavenge(); + ALWAYS_INLINE bool shouldScavenge() const; + +#if !HAVE(DISPATCH_H) + static NO_RETURN_WITH_VALUE void* runScavengerThread(void*); + NO_RETURN void scavengerThread(); + + // Keeps track of whether the background thread is actively scavenging memory every kScavengeDelayInSeconds, or + // it's blocked waiting for more pages to be deleted. + bool m_scavengeThreadActive; + + pthread_mutex_t m_scavengeMutex; + pthread_cond_t m_scavengeCondition; +#else // !HAVE(DISPATCH_H) + void periodicScavenge(); + + dispatch_queue_t m_scavengeQueue; + dispatch_source_t m_scavengeTimer; + bool m_scavengingScheduled; +#endif + +#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY }; void TCMalloc_PageHeap::init() @@ -1142,6 +1465,12 @@ void TCMalloc_PageHeap::init() pagemap_cache_ = PageMapCache(0); free_pages_ = 0; system_bytes_ = 0; + +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + free_committed_pages_ = 0; + min_free_committed_pages_since_last_scavenge_ = 0; +#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + scavenge_counter_ = 0; // Start scavenging at kMaxPages list scavenge_index_ = kMaxPages-1; @@ -1152,8 +1481,99 @@ void TCMalloc_PageHeap::init() DLL_Init(&free_[i].normal); DLL_Init(&free_[i].returned); } + +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + initializeScavenger(); +#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY +} + +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + +#if !HAVE(DISPATCH_H) + +void TCMalloc_PageHeap::initializeScavenger() +{ + pthread_mutex_init(&m_scavengeMutex, 0); + pthread_cond_init(&m_scavengeCondition, 0); + m_scavengeThreadActive = true; + pthread_t thread; + pthread_create(&thread, 0, runScavengerThread, this); +} + +void* TCMalloc_PageHeap::runScavengerThread(void* context) +{ + static_cast(context)->scavengerThread(); +#if COMPILER(MSVC) + // Without this, Visual Studio will complain that this method does not return a value. + return 0; +#endif +} + +ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger() +{ + if (!m_scavengeThreadActive && shouldScavenge()) + pthread_cond_signal(&m_scavengeCondition); +} + +#else // !HAVE(DISPATCH_H) + +void TCMalloc_PageHeap::initializeScavenger() +{ + m_scavengeQueue = dispatch_queue_create("com.apple.JavaScriptCore.FastMallocSavenger", NULL); + m_scavengeTimer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, m_scavengeQueue); + dispatch_time_t startTime = dispatch_time(DISPATCH_TIME_NOW, kScavengeDelayInSeconds * NSEC_PER_SEC); + dispatch_source_set_timer(m_scavengeTimer, startTime, kScavengeDelayInSeconds * NSEC_PER_SEC, 1000 * NSEC_PER_USEC); + dispatch_source_set_event_handler(m_scavengeTimer, ^{ periodicScavenge(); }); + m_scavengingScheduled = false; +} + +ALWAYS_INLINE void TCMalloc_PageHeap::signalScavenger() +{ + if (!m_scavengingScheduled && shouldScavenge()) { + m_scavengingScheduled = true; + dispatch_resume(m_scavengeTimer); + } +} + +#endif + +void TCMalloc_PageHeap::scavenge() +{ + size_t pagesToRelease = min_free_committed_pages_since_last_scavenge_ * kScavengePercentage; + size_t targetPageCount = std::max(kMinimumFreeCommittedPageCount, free_committed_pages_ - pagesToRelease); + + while (free_committed_pages_ > targetPageCount) { + for (int i = kMaxPages; i > 0 && free_committed_pages_ >= targetPageCount; i--) { + SpanList* slist = (static_cast(i) == kMaxPages) ? &large_ : &free_[i]; + // If the span size is bigger than kMinSpanListsWithSpans pages return all the spans in the list, else return all but 1 span. + // Return only 50% of a spanlist at a time so spans of size 1 are not the only ones left. + size_t numSpansToReturn = (i > kMinSpanListsWithSpans) ? DLL_Length(&slist->normal) : static_cast(.5 * DLL_Length(&slist->normal)); + for (int j = 0; static_cast(j) < numSpansToReturn && !DLL_IsEmpty(&slist->normal) && free_committed_pages_ > targetPageCount; j++) { + Span* s = slist->normal.prev; + DLL_Remove(s); + ASSERT(!s->decommitted); + if (!s->decommitted) { + TCMalloc_SystemRelease(reinterpret_cast(s->start << kPageShift), + static_cast(s->length << kPageShift)); + ASSERT(free_committed_pages_ >= s->length); + free_committed_pages_ -= s->length; + s->decommitted = true; + } + DLL_Prepend(&slist->returned, s); + } + } + } + + min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; +} + +ALWAYS_INLINE bool TCMalloc_PageHeap::shouldScavenge() const +{ + return free_committed_pages_ > kMinimumFreeCommittedPageCount; } +#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + inline Span* TCMalloc_PageHeap::New(Length n) { ASSERT(Check()); ASSERT(n > 0); @@ -1176,13 +1596,24 @@ inline Span* TCMalloc_PageHeap::New(Length n) { Span* result = ll->next; Carve(result, n, released); +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + // The newly allocated memory is from a span that's in the normal span list (already committed). Update the + // free committed pages count. + ASSERT(free_committed_pages_ >= n); + free_committed_pages_ -= n; + if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) + min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; +#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY ASSERT(Check()); free_pages_ -= n; return result; } Span* result = AllocLarge(n); - if (result != NULL) return result; + if (result != NULL) { + ASSERT_SPAN_COMMITTED(result); + return result; + } // Grow the heap and try again if (!GrowHeap(n)) { @@ -1229,6 +1660,14 @@ Span* TCMalloc_PageHeap::AllocLarge(Length n) { if (best != NULL) { Carve(best, n, from_released); +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + // The newly allocated memory is from a span that's in the normal span list (already committed). Update the + // free committed pages count. + ASSERT(free_committed_pages_ >= n); + free_committed_pages_ -= n; + if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) + min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; +#endif // USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY ASSERT(Check()); free_pages_ -= n; return best; @@ -1259,17 +1698,28 @@ inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) { span->free = 0; Event(span, 'A', n); + if (released) { + // If the span chosen to carve from is decommited, commit the entire span at once to avoid committing spans 1 page at a time. + ASSERT(span->decommitted); + TCMalloc_SystemCommit(reinterpret_cast(span->start << kPageShift), static_cast(span->length << kPageShift)); + span->decommitted = false; +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + free_committed_pages_ += span->length; +#endif + } + const int extra = static_cast(span->length - n); ASSERT(extra >= 0); if (extra > 0) { Span* leftover = NewSpan(span->start + n, extra); leftover->free = 1; + leftover->decommitted = false; Event(leftover, 'S', extra); RecordSpan(leftover); // Place leftover span on appropriate free list SpanList* listpair = (static_cast(extra) < kMaxPages) ? &free_[extra] : &large_; - Span* dst = released ? &listpair->returned : &listpair->normal; + Span* dst = &listpair->normal; DLL_Prepend(dst, leftover); span->length = n; @@ -1277,6 +1727,18 @@ inline void TCMalloc_PageHeap::Carve(Span* span, Length n, bool released) { } } +static ALWAYS_INLINE void mergeDecommittedStates(Span* destination, Span* other) +{ + if (destination->decommitted && !other->decommitted) { + TCMalloc_SystemRelease(reinterpret_cast(other->start << kPageShift), + static_cast(other->length << kPageShift)); + } else if (other->decommitted && !destination->decommitted) { + TCMalloc_SystemRelease(reinterpret_cast(destination->start << kPageShift), + static_cast(destination->length << kPageShift)); + destination->decommitted = true; + } +} + inline void TCMalloc_PageHeap::Delete(Span* span) { ASSERT(Check()); ASSERT(!span->free); @@ -1284,16 +1746,18 @@ inline void TCMalloc_PageHeap::Delete(Span* span) { ASSERT(GetDescriptor(span->start) == span); ASSERT(GetDescriptor(span->start + span->length - 1) == span); span->sizeclass = 0; +#ifndef NO_TCMALLOC_SAMPLES span->sample = 0; +#endif // Coalesce -- we guarantee that "p" != 0, so no bounds checking // necessary. We do not bother resetting the stale pagemap // entries for the pieces we are merging together because we only // care about the pagemap entries for the boundaries. - // - // Note that the spans we merge into "span" may come out of - // a "returned" list. For simplicity, we move these into the - // "normal" list of the appropriate size class. +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + // Track the total size of the neighboring free spans that are committed. + Length neighboringCommittedSpansLength = 0; +#endif const PageID p = span->start; const Length n = span->length; Span* prev = GetDescriptor(p-1); @@ -1301,6 +1765,11 @@ inline void TCMalloc_PageHeap::Delete(Span* span) { // Merge preceding span into this span ASSERT(prev->start + prev->length == p); const Length len = prev->length; +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + if (!prev->decommitted) + neighboringCommittedSpansLength += len; +#endif + mergeDecommittedStates(span, prev); DLL_Remove(prev); DeleteSpan(prev); span->start -= len; @@ -1313,6 +1782,11 @@ inline void TCMalloc_PageHeap::Delete(Span* span) { // Merge next span into this span ASSERT(next->start == p+n); const Length len = next->length; +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + if (!next->decommitted) + neighboringCommittedSpansLength += len; +#endif + mergeDecommittedStates(span, next); DLL_Remove(next); DeleteSpan(next); span->length += len; @@ -1322,17 +1796,41 @@ inline void TCMalloc_PageHeap::Delete(Span* span) { Event(span, 'D', span->length); span->free = 1; - if (span->length < kMaxPages) { - DLL_Prepend(&free_[span->length].normal, span); + if (span->decommitted) { + if (span->length < kMaxPages) + DLL_Prepend(&free_[span->length].returned, span); + else + DLL_Prepend(&large_.returned, span); } else { - DLL_Prepend(&large_.normal, span); + if (span->length < kMaxPages) + DLL_Prepend(&free_[span->length].normal, span); + else + DLL_Prepend(&large_.normal, span); } free_pages_ += n; +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + if (span->decommitted) { + // If the merged span is decommitted, that means we decommitted any neighboring spans that were + // committed. Update the free committed pages count. + free_committed_pages_ -= neighboringCommittedSpansLength; + if (free_committed_pages_ < min_free_committed_pages_since_last_scavenge_) + min_free_committed_pages_since_last_scavenge_ = free_committed_pages_; + } else { + // If the merged span remains committed, add the deleted span's size to the free committed pages count. + free_committed_pages_ += n; + } + + // Make sure the scavenge thread becomes active if we have enough freed pages to release some back to the system. + signalScavenger(); +#else IncrementalScavenge(n); +#endif + ASSERT(Check()); } +#if !USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY void TCMalloc_PageHeap::IncrementalScavenge(Length n) { // Fast path; not yet time to release memory scavenge_counter_ -= n; @@ -1351,6 +1849,7 @@ void TCMalloc_PageHeap::IncrementalScavenge(Length n) { DLL_Remove(s); TCMalloc_SystemRelease(reinterpret_cast(s->start << kPageShift), static_cast(s->length << kPageShift)); + s->decommitted = true; DLL_Prepend(&slist->returned, s); scavenge_counter_ = std::max(16UL, std::min(kDefaultReleaseDelay, kDefaultReleaseDelay - (free_pages_ / kDefaultReleaseDelay))); @@ -1367,6 +1866,7 @@ void TCMalloc_PageHeap::IncrementalScavenge(Length n) { // Nothing to scavenge, delay for a while scavenge_counter_ = kDefaultReleaseDelay; } +#endif void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) { // Associate span object with all interior pages as well @@ -1379,6 +1879,21 @@ void TCMalloc_PageHeap::RegisterSizeClass(Span* span, size_t sc) { pagemap_.set(span->start+i, span); } } + +#ifdef WTF_CHANGES +size_t TCMalloc_PageHeap::ReturnedBytes() const { + size_t result = 0; + for (unsigned s = 0; s < kMaxPages; s++) { + const int r_length = DLL_Length(&free_[s].returned); + unsigned r_pages = s * r_length; + result += r_pages << kPageShift; + } + + for (Span* s = large_.returned.next; s != &large_.returned; s = s->next) + result += s->length << kPageShift; + return result; +} +#endif #ifndef WTF_CHANGES static double PagesToMB(uint64_t pages) { @@ -1457,7 +1972,7 @@ bool TCMalloc_PageHeap::GrowHeap(Length n) { if (n < ask) { // Try growing just "n" pages ask = n; - ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);; + ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize); } if (ptr == NULL) return false; } @@ -1652,10 +2167,10 @@ class TCMalloc_ThreadCache { // Total byte size in cache size_t Size() const { return size_; } - void* Allocate(size_t size); + ALWAYS_INLINE void* Allocate(size_t size); void Deallocate(void* ptr, size_t size_class); - void FetchFromCentralCache(size_t cl, size_t allocationSize); + ALWAYS_INLINE void FetchFromCentralCache(size_t cl, size_t allocationSize); void ReleaseToCentralCache(size_t cl, int N); void Scavenge(); void Print() const; @@ -1718,13 +2233,18 @@ class TCMalloc_Central_FreeList { #ifdef WTF_CHANGES template - void enumerateFreeObjects(Finder& finder, const Reader& reader) + void enumerateFreeObjects(Finder& finder, const Reader& reader, TCMalloc_Central_FreeList* remoteCentralFreeList) { for (Span* span = &empty_; span && span != &empty_; span = (span->next ? reader(span->next) : 0)) ASSERT(!span->objects); ASSERT(!nonempty_.objects); - for (Span* span = reader(nonempty_.next); span && span != &nonempty_; span = (span->next ? reader(span->next) : 0)) { + static const ptrdiff_t nonemptyOffset = reinterpret_cast(&nonempty_) - reinterpret_cast(this); + + Span* remoteNonempty = reinterpret_cast(reinterpret_cast(remoteCentralFreeList) + nonemptyOffset); + Span* remoteSpan = nonempty_.next; + + for (Span* span = reader(remoteSpan); span && remoteSpan != remoteNonempty; remoteSpan = span->next, span = (span->next ? reader(span->next) : 0)) { for (void* nextObject = span->objects; nextObject; nextObject = *reader(reinterpret_cast(nextObject))) finder.visit(nextObject); } @@ -1751,12 +2271,12 @@ class TCMalloc_Central_FreeList { // REQUIRES: lock_ is held // Release an object to spans. // May temporarily release lock_. - void ReleaseToSpans(void* object); + ALWAYS_INLINE void ReleaseToSpans(void* object); // REQUIRES: lock_ is held // Populate cache by fetching from the page heap. // May temporarily release lock_. - void Populate(); + ALWAYS_INLINE void Populate(); // REQUIRES: lock is held. // Tries to make room for a TCEntry. If the cache is full it will try to @@ -1769,7 +2289,7 @@ class TCMalloc_Central_FreeList { // just iterates over the sizeclasses but does so without taking a lock. // Returns true on success. // May temporarily lock a "random" size class. - static bool EvictRandomSizeClass(size_t locked_size_class, bool force); + static ALWAYS_INLINE bool EvictRandomSizeClass(size_t locked_size_class, bool force); // REQUIRES: lock_ is *not* held. // Tries to shrink the Cache. If force is true it will relase objects to @@ -1824,7 +2344,7 @@ static SpinLock pageheap_lock = SPINLOCK_INITIALIZER; #if PLATFORM(ARM) static void* pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(void*) - 1) / sizeof(void*)] __attribute__((aligned)); #else -static void* pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(void*) - 1) / sizeof(void*)]; +static AllocAlignmentInteger pageheap_memory[(sizeof(TCMalloc_PageHeap) + sizeof(AllocAlignmentInteger) - 1) / sizeof(AllocAlignmentInteger)]; #endif static bool phinited = false; @@ -1843,6 +2363,57 @@ static inline TCMalloc_PageHeap* getPageHeap() #define pageheap getPageHeap() +#if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY + +#if !HAVE(DISPATCH_H) +#if OS(WINDOWS) +static void sleep(unsigned seconds) +{ + ::Sleep(seconds * 1000); +} +#endif + +void TCMalloc_PageHeap::scavengerThread() +{ +#if HAVE(PTHREAD_SETNAME_NP) + pthread_setname_np("JavaScriptCore: FastMalloc scavenger"); +#endif + + while (1) { + if (!shouldScavenge()) { + pthread_mutex_lock(&m_scavengeMutex); + m_scavengeThreadActive = false; + // Block until there are enough free committed pages to release back to the system. + pthread_cond_wait(&m_scavengeCondition, &m_scavengeMutex); + m_scavengeThreadActive = true; + pthread_mutex_unlock(&m_scavengeMutex); + } + sleep(kScavengeDelayInSeconds); + { + SpinLockHolder h(&pageheap_lock); + pageheap->scavenge(); + } + } +} + +#else + +void TCMalloc_PageHeap::periodicScavenge() +{ + { + SpinLockHolder h(&pageheap_lock); + pageheap->scavenge(); + } + + if (!shouldScavenge()) { + m_scavengingScheduled = false; + dispatch_suspend(m_scavengeTimer); + } +} +#endif // HAVE(DISPATCH_H) + +#endif + // If TLS is available, we also store a copy // of the per-thread object in a __thread variable // since __thread variables are faster to read @@ -1859,7 +2430,11 @@ static __thread TCMalloc_ThreadCache *threadlocal_heap; // Therefore, we use TSD keys only after tsd_inited is set to true. // Until then, we use a slow path to get the heap object. static bool tsd_inited = false; +#if USE(PTHREAD_GETSPECIFIC_DIRECT) +static pthread_key_t heap_key = __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY0; +#else static pthread_key_t heap_key; +#endif #if COMPILER(MSVC) DWORD tlsIndex = TLS_OUT_OF_INDEXES; #endif @@ -1930,7 +2505,7 @@ ALWAYS_INLINE void TCMalloc_Central_FreeList::ReleaseToSpans(void* object) { // The following check is expensive, so it is disabled by default if (false) { // Check that object does not occur in list - int got = 0; + unsigned got = 0; for (void* p = span->objects; p != NULL; p = *((void**) p)) { ASSERT(p != object); got++; @@ -2096,6 +2671,7 @@ void* TCMalloc_Central_FreeList::FetchFromSpans() { Span* span = nonempty_.next; ASSERT(span->objects != NULL); + ASSERT_SPAN_COMMITTED(span); span->refcount++; void* result = span->objects; span->objects = *(reinterpret_cast(result)); @@ -2126,6 +2702,7 @@ ALWAYS_INLINE void TCMalloc_Central_FreeList::Populate() { lock_.Lock(); return; } + ASSERT_SPAN_COMMITTED(span); ASSERT(span->length == npages); // Cache sizeclass info eagerly. Locking is not necessary. // (Instead of being eager, we could just replace any stale info @@ -2348,7 +2925,7 @@ void TCMalloc_ThreadCache::InitModule() { } pageheap->init(); phinited = 1; -#if defined(WTF_CHANGES) && PLATFORM(DARWIN) +#if defined(WTF_CHANGES) && OS(DARWIN) FastMallocZone::init(); #endif } @@ -2401,7 +2978,11 @@ inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetCacheIfPresent() { void TCMalloc_ThreadCache::InitTSD() { ASSERT(!tsd_inited); +#if USE(PTHREAD_GETSPECIFIC_DIRECT) + pthread_key_init_np(heap_key, DestroyThreadCache); +#else pthread_key_create(&heap_key, DestroyThreadCache); +#endif #if COMPILER(MSVC) tlsIndex = TlsAlloc(); #endif @@ -2902,11 +3483,15 @@ static inline void* CheckedMallocResult(void *result) } static inline void* SpanToMallocResult(Span *span) { + ASSERT_SPAN_COMMITTED(span); pageheap->CacheSizeClass(span->start, 0); return CheckedMallocResult(reinterpret_cast(span->start << kPageShift)); } +#ifdef WTF_CHANGES +template +#endif static ALWAYS_INLINE void* do_malloc(size_t size) { void* ret = NULL; @@ -2936,7 +3521,14 @@ static ALWAYS_INLINE void* do_malloc(size_t size) { // size-appropriate freelist, afer replenishing it if it's empty. ret = CheckedMallocResult(heap->Allocate(size)); } - if (ret == NULL) errno = ENOMEM; + if (!ret) { +#ifdef WTF_CHANGES + if (crashOnFailure) // This branch should be optimized out by the compiler. + CRASH(); +#else + errno = ENOMEM; +#endif + } return ret; } @@ -2953,7 +3545,9 @@ static ALWAYS_INLINE void do_free(void* ptr) { pageheap->CacheSizeClass(p, cl); } if (cl != 0) { +#ifndef NO_TCMALLOC_SAMPLES ASSERT(!pageheap->GetDescriptor(p)->sample); +#endif TCMalloc_ThreadCache* heap = TCMalloc_ThreadCache::GetCacheIfPresent(); if (heap != NULL) { heap->Deallocate(ptr, cl); @@ -2966,11 +3560,13 @@ static ALWAYS_INLINE void do_free(void* ptr) { SpinLockHolder h(&pageheap_lock); ASSERT(reinterpret_cast(ptr) % kPageSize == 0); ASSERT(span != NULL && span->start == p); +#ifndef NO_TCMALLOC_SAMPLES if (span->sample) { DLL_Remove(span); stacktrace_allocator.Delete(reinterpret_cast(span->objects)); span->objects = NULL; } +#endif pageheap->Delete(span); } } @@ -3096,9 +3692,40 @@ static inline struct mallinfo do_mallinfo() { #ifndef WTF_CHANGES extern "C" +#else +#define do_malloc do_malloc + +template +ALWAYS_INLINE void* malloc(size_t); + +void* fastMalloc(size_t size) +{ + return malloc(size); +} + +TryMallocReturnValue tryFastMalloc(size_t size) +{ + return malloc(size); +} + +template +ALWAYS_INLINE #endif void* malloc(size_t size) { - void* result = do_malloc(size); +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (std::numeric_limits::max() - sizeof(AllocAlignmentInteger) <= size) // If overflow would occur... + return 0; + size += sizeof(AllocAlignmentInteger); + void* result = do_malloc(size); + if (!result) + return 0; + + *static_cast(result) = Internal::AllocTypeMalloc; + result = static_cast(result) + 1; +#else + void* result = do_malloc(size); +#endif + #ifndef WTF_CHANGES MallocHook::InvokeNewHook(result, size); #endif @@ -3112,29 +3739,73 @@ void free(void* ptr) { #ifndef WTF_CHANGES MallocHook::InvokeDeleteHook(ptr); #endif - do_free(ptr); + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (!ptr) + return; + + AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(ptr); + if (*header != Internal::AllocTypeMalloc) + Internal::fastMallocMatchFailed(ptr); + do_free(header); +#else + do_free(ptr); +#endif } #ifndef WTF_CHANGES extern "C" +#else +template +ALWAYS_INLINE void* calloc(size_t, size_t); + +void* fastCalloc(size_t n, size_t elem_size) +{ + return calloc(n, elem_size); +} + +TryMallocReturnValue tryFastCalloc(size_t n, size_t elem_size) +{ + return calloc(n, elem_size); +} + +template +ALWAYS_INLINE #endif void* calloc(size_t n, size_t elem_size) { - const size_t totalBytes = n * elem_size; + size_t totalBytes = n * elem_size; // Protect against overflow if (n > 1 && elem_size && (totalBytes / elem_size) != n) return 0; - - void* result = do_malloc(totalBytes); - if (result != NULL) { + +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (std::numeric_limits::max() - sizeof(AllocAlignmentInteger) <= totalBytes) // If overflow would occur... + return 0; + + totalBytes += sizeof(AllocAlignmentInteger); + void* result = do_malloc(totalBytes); + if (!result) + return 0; + memset(result, 0, totalBytes); - } + *static_cast(result) = Internal::AllocTypeMalloc; + result = static_cast(result) + 1; +#else + void* result = do_malloc(totalBytes); + if (result != NULL) { + memset(result, 0, totalBytes); + } +#endif + #ifndef WTF_CHANGES MallocHook::InvokeNewHook(result, totalBytes); #endif return result; } +// Since cfree isn't used anywhere, we don't compile it in. +#ifndef WTF_CHANGES #ifndef WTF_CHANGES extern "C" #endif @@ -3144,15 +3815,36 @@ void cfree(void* ptr) { #endif do_free(ptr); } +#endif #ifndef WTF_CHANGES extern "C" +#else +template +ALWAYS_INLINE void* realloc(void*, size_t); + +void* fastRealloc(void* old_ptr, size_t new_size) +{ + return realloc(old_ptr, new_size); +} + +TryMallocReturnValue tryFastRealloc(void* old_ptr, size_t new_size) +{ + return realloc(old_ptr, new_size); +} + +template +ALWAYS_INLINE #endif void* realloc(void* old_ptr, size_t new_size) { if (old_ptr == NULL) { +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + void* result = malloc(new_size); +#else void* result = do_malloc(new_size); #ifndef WTF_CHANGES MallocHook::InvokeNewHook(result, new_size); +#endif #endif return result; } @@ -3164,6 +3856,16 @@ void* realloc(void* old_ptr, size_t new_size) { return NULL; } +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + if (std::numeric_limits::max() - sizeof(AllocAlignmentInteger) <= new_size) // If overflow would occur... + return 0; + new_size += sizeof(AllocAlignmentInteger); + AllocAlignmentInteger* header = Internal::fastMallocMatchValidationValue(old_ptr); + if (*header != Internal::AllocTypeMalloc) + Internal::fastMallocMatchFailed(old_ptr); + old_ptr = header; +#endif + // Get the size of the old entry const PageID p = reinterpret_cast(old_ptr) >> kPageShift; size_t cl = pageheap->GetSizeClassIfCached(p); @@ -3200,13 +3902,21 @@ void* realloc(void* old_ptr, size_t new_size) { // that we already know the sizeclass of old_ptr. The benefit // would be small, so don't bother. do_free(old_ptr); +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + new_ptr = static_cast(new_ptr) + 1; +#endif return new_ptr; } else { +#if ENABLE(FAST_MALLOC_MATCH_VALIDATION) + old_ptr = static_cast(old_ptr) + 1; // Set old_ptr back to the user pointer. +#endif return old_ptr; } } -#ifndef WTF_CHANGES +#ifdef WTF_CHANGES +#undef do_malloc +#else static SpinLock set_new_handler_lock = SPINLOCK_INITIALIZER; @@ -3248,6 +3958,8 @@ static inline void* cpp_alloc(size_t size, bool nothrow) { } } +#if ENABLE(GLOBAL_FASTMALLOC_NEW) + void* operator new(size_t size) { void* p = cpp_alloc(size, false); // We keep this next instruction out of cpp_alloc for a reason: when @@ -3302,6 +4014,8 @@ void operator delete[](void* p, const std::nothrow_t&) __THROW { do_free(p); } +#endif + extern "C" void* memalign(size_t align, size_t size) __THROW { void* result = do_memalign(align, size); MallocHook::InvokeNewHook(result, size); @@ -3369,7 +4083,7 @@ extern "C" struct mallinfo mallinfo(void) { #if defined(__GLIBC__) extern "C" { -# if defined(__GNUC__) && !defined(__MACH__) && defined(HAVE___ATTRIBUTE__) +#if COMPILER(GCC) && !defined(__MACH__) && defined(HAVE___ATTRIBUTE__) // Potentially faster variants that use the gcc alias extension. // Mach-O (Darwin) does not support weak aliases, hence the __MACH__ check. # define ALIAS(x) __attribute__ ((weak, alias (x))) @@ -3417,8 +4131,62 @@ void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride; #endif -#if defined(WTF_CHANGES) && PLATFORM(DARWIN) -#include +#ifdef WTF_CHANGES +void releaseFastMallocFreeMemory() +{ + // Flush free pages in the current thread cache back to the page heap. + // Low watermark mechanism in Scavenge() prevents full return on the first pass. + // The second pass flushes everything. + if (TCMalloc_ThreadCache* threadCache = TCMalloc_ThreadCache::GetCacheIfPresent()) { + threadCache->Scavenge(); + threadCache->Scavenge(); + } + + SpinLockHolder h(&pageheap_lock); + pageheap->ReleaseFreePages(); +} + +FastMallocStatistics fastMallocStatistics() +{ + FastMallocStatistics statistics; + + SpinLockHolder lockHolder(&pageheap_lock); + statistics.reservedVMBytes = static_cast(pageheap->SystemBytes()); + statistics.committedVMBytes = statistics.reservedVMBytes - pageheap->ReturnedBytes(); + + statistics.freeListBytes = 0; + for (unsigned cl = 0; cl < kNumClasses; ++cl) { + const int length = central_cache[cl].length(); + const int tc_length = central_cache[cl].tc_length(); + + statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length); + } + for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadCache = threadCache->next_) + statistics.freeListBytes += threadCache->Size(); + + return statistics; +} + +size_t fastMallocSize(const void* ptr) +{ + const PageID p = reinterpret_cast(ptr) >> kPageShift; + Span* span = pageheap->GetDescriptorEnsureSafe(p); + + if (!span || span->free) + return 0; + + for (void* free = span->objects; free != NULL; free = *((void**) free)) { + if (ptr == free) + return 0; + } + + if (size_t cl = span->sizeclass) + return ByteSizeForClass(cl); + + return span->length << kPageShift; +} + +#if OS(DARWIN) class FreeObjectFinder { const RemoteMemoryReader& m_reader; @@ -3429,6 +4197,7 @@ public: void visit(void* ptr) { m_freeObjects.add(ptr); } bool isFreeObject(void* ptr) const { return m_freeObjects.contains(ptr); } + bool isFreeObject(vm_address_t ptr) const { return isFreeObject(reinterpret_cast(ptr)); } size_t freeObjectCount() const { return m_freeObjects.size(); } void findFreeObjects(TCMalloc_ThreadCache* threadCache) @@ -3437,10 +4206,10 @@ public: threadCache->enumerateFreeObjects(*this, m_reader); } - void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes) + void findFreeObjects(TCMalloc_Central_FreeListPadded* centralFreeList, size_t numSizes, TCMalloc_Central_FreeListPadded* remoteCentralFreeList) { for (unsigned i = 0; i < numSizes; i++) - centralFreeList[i].enumerateFreeObjects(*this, m_reader); + centralFreeList[i].enumerateFreeObjects(*this, m_reader, remoteCentralFreeList + i); } }; @@ -3479,7 +4248,9 @@ class PageMapMemoryUsageRecorder { vm_range_recorder_t* m_recorder; const RemoteMemoryReader& m_reader; const FreeObjectFinder& m_freeObjectFinder; - mutable HashSet m_seenPointers; + + HashSet m_seenPointers; + Vector m_coalescedSpans; public: PageMapMemoryUsageRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader, const FreeObjectFinder& freeObjectFinder) @@ -3491,51 +4262,133 @@ public: , m_freeObjectFinder(freeObjectFinder) { } - int visit(void* ptr) const + ~PageMapMemoryUsageRecorder() + { + ASSERT(!m_coalescedSpans.size()); + } + + void recordPendingRegions() + { + Span* lastSpan = m_coalescedSpans[m_coalescedSpans.size() - 1]; + vm_range_t ptrRange = { m_coalescedSpans[0]->start << kPageShift, 0 }; + ptrRange.size = (lastSpan->start << kPageShift) - ptrRange.address + (lastSpan->length * kPageSize); + + // Mark the memory region the spans represent as a candidate for containing pointers + if (m_typeMask & MALLOC_PTR_REGION_RANGE_TYPE) + (*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1); + + if (!(m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) { + m_coalescedSpans.clear(); + return; + } + + Vector allocatedPointers; + for (size_t i = 0; i < m_coalescedSpans.size(); ++i) { + Span *theSpan = m_coalescedSpans[i]; + if (theSpan->free) + continue; + + vm_address_t spanStartAddress = theSpan->start << kPageShift; + vm_size_t spanSizeInBytes = theSpan->length * kPageSize; + + if (!theSpan->sizeclass) { + // If it's an allocated large object span, mark it as in use + if (!m_freeObjectFinder.isFreeObject(spanStartAddress)) + allocatedPointers.append((vm_range_t){spanStartAddress, spanSizeInBytes}); + } else { + const size_t objectSize = ByteSizeForClass(theSpan->sizeclass); + + // Mark each allocated small object within the span as in use + const vm_address_t endOfSpan = spanStartAddress + spanSizeInBytes; + for (vm_address_t object = spanStartAddress; object + objectSize <= endOfSpan; object += objectSize) { + if (!m_freeObjectFinder.isFreeObject(object)) + allocatedPointers.append((vm_range_t){object, objectSize}); + } + } + } + + (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, allocatedPointers.data(), allocatedPointers.size()); + + m_coalescedSpans.clear(); + } + + int visit(void* ptr) { if (!ptr) return 1; Span* span = m_reader(reinterpret_cast(ptr)); + if (!span->start) + return 1; + if (m_seenPointers.contains(ptr)) return span->length; m_seenPointers.add(ptr); - // Mark the memory used for the Span itself as an administrative region - vm_range_t ptrRange = { reinterpret_cast(ptr), sizeof(Span) }; - if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE)) - (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, &ptrRange, 1); + if (!m_coalescedSpans.size()) { + m_coalescedSpans.append(span); + return span->length; + } - ptrRange.address = span->start << kPageShift; - ptrRange.size = span->length * kPageSize; + Span* previousSpan = m_coalescedSpans[m_coalescedSpans.size() - 1]; + vm_address_t previousSpanStartAddress = previousSpan->start << kPageShift; + vm_size_t previousSpanSizeInBytes = previousSpan->length * kPageSize; - // Mark the memory region the span represents as candidates for containing pointers - if (m_typeMask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE)) - (*m_recorder)(m_task, m_context, MALLOC_PTR_REGION_RANGE_TYPE, &ptrRange, 1); + // If the new span is adjacent to the previous span, do nothing for now. + vm_address_t spanStartAddress = span->start << kPageShift; + if (spanStartAddress == previousSpanStartAddress + previousSpanSizeInBytes) { + m_coalescedSpans.append(span); + return span->length; + } - if (!span->free && (m_typeMask & MALLOC_PTR_IN_USE_RANGE_TYPE)) { - // If it's an allocated large object span, mark it as in use - if (span->sizeclass == 0 && !m_freeObjectFinder.isFreeObject(reinterpret_cast(ptrRange.address))) - (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &ptrRange, 1); - else if (span->sizeclass) { - const size_t byteSize = ByteSizeForClass(span->sizeclass); - unsigned totalObjects = (span->length << kPageShift) / byteSize; - ASSERT(span->refcount <= totalObjects); - char* ptr = reinterpret_cast(span->start << kPageShift); + // New span is not adjacent to previous span, so record the spans coalesced so far. + recordPendingRegions(); + m_coalescedSpans.append(span); - // Mark each allocated small object within the span as in use - for (unsigned i = 0; i < totalObjects; i++) { - char* thisObject = ptr + (i * byteSize); - if (m_freeObjectFinder.isFreeObject(thisObject)) - continue; + return span->length; + } +}; - vm_range_t objectRange = { reinterpret_cast(thisObject), byteSize }; - (*m_recorder)(m_task, m_context, MALLOC_PTR_IN_USE_RANGE_TYPE, &objectRange, 1); - } - } +class AdminRegionRecorder { + task_t m_task; + void* m_context; + unsigned m_typeMask; + vm_range_recorder_t* m_recorder; + const RemoteMemoryReader& m_reader; + + Vector m_pendingRegions; + +public: + AdminRegionRecorder(task_t task, void* context, unsigned typeMask, vm_range_recorder_t* recorder, const RemoteMemoryReader& reader) + : m_task(task) + , m_context(context) + , m_typeMask(typeMask) + , m_recorder(recorder) + , m_reader(reader) + { } + + void recordRegion(vm_address_t ptr, size_t size) + { + if (m_typeMask & MALLOC_ADMIN_REGION_RANGE_TYPE) + m_pendingRegions.append((vm_range_t){ ptr, size }); + } + + void visit(void *ptr, size_t size) + { + recordRegion(reinterpret_cast(ptr), size); + } + + void recordPendingRegions() + { + if (m_pendingRegions.size()) { + (*m_recorder)(m_task, m_context, MALLOC_ADMIN_REGION_RANGE_TYPE, m_pendingRegions.data(), m_pendingRegions.size()); + m_pendingRegions.clear(); } + } - return span->length; + ~AdminRegionRecorder() + { + ASSERT(!m_pendingRegions.size()); } }; @@ -3554,14 +4407,26 @@ kern_return_t FastMallocZone::enumerate(task_t task, void* context, unsigned typ FreeObjectFinder finder(memoryReader); finder.findFreeObjects(threadHeaps); - finder.findFreeObjects(centralCaches, kNumClasses); + finder.findFreeObjects(centralCaches, kNumClasses, mzone->m_centralCaches); TCMalloc_PageHeap::PageMap* pageMap = &pageHeap->pagemap_; PageMapFreeObjectFinder pageMapFinder(memoryReader, finder); - pageMap->visit(pageMapFinder, memoryReader); + pageMap->visitValues(pageMapFinder, memoryReader); PageMapMemoryUsageRecorder usageRecorder(task, context, typeMask, recorder, memoryReader, finder); - pageMap->visit(usageRecorder, memoryReader); + pageMap->visitValues(usageRecorder, memoryReader); + usageRecorder.recordPendingRegions(); + + AdminRegionRecorder adminRegionRecorder(task, context, typeMask, recorder, memoryReader); + pageMap->visitAllocations(adminRegionRecorder, memoryReader); + + PageHeapAllocator* spanAllocator = memoryReader(mzone->m_spanAllocator); + PageHeapAllocator* pageHeapAllocator = memoryReader(mzone->m_pageHeapAllocator); + + spanAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader); + pageHeapAllocator->recordAdministrativeRegions(adminRegionRecorder, memoryReader); + + adminRegionRecorder.recordPendingRegions(); return 0; } @@ -3602,15 +4467,22 @@ void* FastMallocZone::zoneRealloc(malloc_zone_t*, void*, size_t) extern "C" { malloc_introspection_t jscore_fastmalloc_introspection = { &FastMallocZone::enumerate, &FastMallocZone::goodSize, &FastMallocZone::check, &FastMallocZone::print, - &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics }; + &FastMallocZone::log, &FastMallocZone::forceLock, &FastMallocZone::forceUnlock, &FastMallocZone::statistics + + , 0 // zone_locked will not be called on the zone unless it advertises itself as version five or higher. + + }; } -FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches) +FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache** threadHeaps, TCMalloc_Central_FreeListPadded* centralCaches, PageHeapAllocator* spanAllocator, PageHeapAllocator* pageHeapAllocator) : m_pageHeap(pageHeap) , m_threadHeaps(threadHeaps) , m_centralCaches(centralCaches) + , m_spanAllocator(spanAllocator) + , m_pageHeapAllocator(pageHeapAllocator) { memset(&m_zone, 0, sizeof(m_zone)); + m_zone.version = 4; m_zone.zone_name = "JavaScriptCore FastMalloc"; m_zone.size = &FastMallocZone::size; m_zone.malloc = &FastMallocZone::zoneMalloc; @@ -3626,19 +4498,12 @@ FastMallocZone::FastMallocZone(TCMalloc_PageHeap* pageHeap, TCMalloc_ThreadCache void FastMallocZone::init() { - static FastMallocZone zone(pageheap, &thread_heaps, static_cast(central_cache)); + static FastMallocZone zone(pageheap, &thread_heaps, static_cast(central_cache), &span_allocator, &threadheap_allocator); } -#endif +#endif // OS(DARWIN) -void releaseFastMallocFreeMemory() -{ - SpinLockHolder h(&pageheap_lock); - pageheap->ReleaseFreePages(); -} - -#if WTF_CHANGES } // namespace WTF -#endif +#endif // WTF_CHANGES #endif // FORCE_SYSTEM_MALLOC