JavaScriptCore-7600.1.4.11.8.tar.gz

[apple/javascriptcore.git] / jit / ExecutableAllocatorFixedVMPool.cpp

diff --git a/jit/ExecutableAllocatorFixedVMPool.cpp b/jit/ExecutableAllocatorFixedVMPool.cpp
index 3048e20cdc8d5da0c4f1658a6680542c7e5e9fca..9afb055d47a0545d833bdadfcdb24acbae6bb1a7 100644 (file)
--- a/jit/ExecutableAllocatorFixedVMPool.cpp
+++ b/jit/ExecutableAllocatorFixedVMPool.cpp
@@ -24,531 +24,169 @@
  */
 
 #include "config.h"
  */
 
 #include "config.h"

-

 #include "ExecutableAllocator.h"
 
 #include "ExecutableAllocator.h"
 

+#include "JSCInlines.h"
+

 #if ENABLE(EXECUTABLE_ALLOCATOR_FIXED)
 
 #if ENABLE(EXECUTABLE_ALLOCATOR_FIXED)
 

+#include "CodeProfiling.h"

 #include <errno.h>
 #include <errno.h>

-
-#include "TCSpinLock.h"
-#include <sys/mman.h>

 #include <unistd.h>
 #include <unistd.h>

-#include <wtf/AVLTree.h>
+#include <wtf/MetaAllocator.h>
+#include <wtf/PageReservation.h>

 #include <wtf/VMTags.h>
 
 #include <wtf/VMTags.h>
 

-    #define MMAP_FLAGS (MAP_PRIVATE | MAP_ANON | MAP_JIT)
-
-using namespace WTF;
-
-namespace JSC {
-
-#define TwoPow(n) (1ull << n)
-
-class AllocationTableSizeClass {
-public:
-    AllocationTableSizeClass(size_t size, size_t blockSize, unsigned log2BlockSize)
-        : m_blockSize(blockSize)
-    {
-        ASSERT(blockSize == TwoPow(log2BlockSize));
-
-        // Calculate the number of blocks needed to hold size.
-        size_t blockMask = blockSize - 1;
-        m_blockCount = (size + blockMask) >> log2BlockSize;
-
-        // Align to the smallest power of two >= m_blockCount.
-        m_blockAlignment = 1;
-        while (m_blockAlignment < m_blockCount)
-            m_blockAlignment += m_blockAlignment;
-    }
-
-    size_t blockSize() const { return m_blockSize; }
-    size_t blockCount() const { return m_blockCount; }
-    size_t blockAlignment() const { return m_blockAlignment; }
-
-    size_t size()
-    {
-        return m_blockSize * m_blockCount;
-    }
-
-private:
-    size_t m_blockSize;
-    size_t m_blockCount;
-    size_t m_blockAlignment;
-};
-
-template<unsigned log2Entries>
-class AllocationTableLeaf {
-    typedef uint64_t BitField;
-
-public:
-    static const unsigned log2SubregionSize = 12; // 2^12 == pagesize
-    static const unsigned log2RegionSize = log2SubregionSize + log2Entries;
-
-    static const size_t subregionSize = TwoPow(log2SubregionSize);
-    static const size_t regionSize = TwoPow(log2RegionSize);
-    static const unsigned entries = TwoPow(log2Entries);
-    COMPILE_ASSERT(entries <= (sizeof(BitField) * 8), AllocationTableLeaf_entries_fit_in_BitField);
-
-    AllocationTableLeaf()
-        : m_allocated(0)
-    {
-    }
-
-    ~AllocationTableLeaf()
-    {
-        ASSERT(isEmpty());
-    }
-
-    size_t allocate(AllocationTableSizeClass& sizeClass)
-    {
-        ASSERT(sizeClass.blockSize() == subregionSize);
-        ASSERT(!isFull());
-
-        size_t alignment = sizeClass.blockAlignment();
-        size_t count = sizeClass.blockCount();
-        // Use this mask to check for spans of free blocks.
-        BitField mask = ((1ull << count) - 1) << (alignment - count);
-
-        // Step in units of alignment size.
-        for (unsigned i = 0; i < entries; i += alignment) {
-            if (!(m_allocated & mask)) {
-                m_allocated |= mask;
-                return (i + (alignment - count)) << log2SubregionSize;
-            }
-            mask <<= alignment;
-        }
-        return notFound;
-    }
-
-    void free(size_t location, AllocationTableSizeClass& sizeClass)
-    {
-        ASSERT(sizeClass.blockSize() == subregionSize);
-
-        size_t entry = location >> log2SubregionSize;
-        size_t count = sizeClass.blockCount();
-        BitField mask = ((1ull << count) - 1) << entry;
-
-        ASSERT((m_allocated & mask) == mask);
-        m_allocated &= ~mask;
-    }
-
-    bool isEmpty()
-    {
-        return !m_allocated;
-    }
-
-    bool isFull()
-    {
-        return !~m_allocated;
-    }
-
-    static size_t size()
-    {
-        return regionSize;
-    }
+#if OS(DARWIN)
+#include <sys/mman.h>
+#endif

 
 

-    static AllocationTableSizeClass classForSize(size_t size)
-    {
-        return AllocationTableSizeClass(size, subregionSize, log2SubregionSize);
-    }
+#if OS(LINUX)
+#include <stdio.h>
+#endif

 
 

-#ifndef NDEBUG
-    void dump(size_t parentOffset = 0, unsigned indent = 0)
-    {
-        for (unsigned i = 0; i < indent; ++i)
-            fprintf(stderr, "    ");
-        fprintf(stderr, "%08x: [%016llx]\n", (int)parentOffset, m_allocated);
-    }
+#if PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED < 1090
+// MADV_FREE_REUSABLE does not work for JIT memory on older OSes so use MADV_FREE in that case.
+#define WTF_USE_MADV_FREE_FOR_JIT_MEMORY 1

 #endif
 
 #endif
 

-private:
-    BitField m_allocated;
-};
+using namespace WTF;

 
 

+namespace JSC {
+    
+uintptr_t startOfFixedExecutableMemoryPool;

 
 

-template<class NextLevel>
-class LazyAllocationTable {
+class FixedVMPoolExecutableAllocator : public MetaAllocator {
+    WTF_MAKE_FAST_ALLOCATED;

 public:
 public:

-    static const unsigned log2RegionSize = NextLevel::log2RegionSize;
-    static const unsigned entries = NextLevel::entries;
-
-    LazyAllocationTable()
-        : m_ptr(0)
-    {
-    }
-
-    ~LazyAllocationTable()
-    {
-        ASSERT(isEmpty());
-    }
-
-    size_t allocate(AllocationTableSizeClass& sizeClass)
-    {
-        if (!m_ptr)
-            m_ptr = new NextLevel();
-        return m_ptr->allocate(sizeClass);
-    }
-
-    void free(size_t location, AllocationTableSizeClass& sizeClass)
-    {
-        ASSERT(m_ptr);
-        m_ptr->free(location, sizeClass);
-        if (m_ptr->isEmpty()) {
-            delete m_ptr;
-            m_ptr = 0;
+    FixedVMPoolExecutableAllocator()
+        : MetaAllocator(jitAllocationGranule) // round up all allocations to 32 bytes
+    {
+        m_reservation = PageReservation::reserveWithGuardPages(fixedExecutableMemoryPoolSize, OSAllocator::JSJITCodePages, EXECUTABLE_POOL_WRITABLE, true);
+        if (m_reservation) {
+            ASSERT(m_reservation.size() == fixedExecutableMemoryPoolSize);
+            addFreshFreeSpace(m_reservation.base(), m_reservation.size());
+            
+            startOfFixedExecutableMemoryPool = reinterpret_cast<uintptr_t>(m_reservation.base());

         }
     }
 
         }
     }
 

-    bool isEmpty()
-    {
-        return !m_ptr;
-    }
-
-    bool isFull()
-    {
-        return m_ptr && m_ptr->isFull();
-    }
-
-    static size_t size()
+    virtual ~FixedVMPoolExecutableAllocator();
+    
+protected:
+    virtual void* allocateNewSpace(size_t&) override

     {
     {

-        return NextLevel::size();
+        // We're operating in a fixed pool, so new allocation is always prohibited.
+        return 0;

     }
     }

-
-#ifndef NDEBUG
-    void dump(size_t parentOffset = 0, unsigned indent = 0)
+    
+    virtual void notifyNeedPage(void* page) override

     {
     {

-        ASSERT(m_ptr);
-        m_ptr->dump(parentOffset, indent);
-    }
+#if USE(MADV_FREE_FOR_JIT_MEMORY)
+        UNUSED_PARAM(page);
+#else
+        m_reservation.commit(page, pageSize());

 #endif
 #endif

-
-    static AllocationTableSizeClass classForSize(size_t size)
-    {
-        return NextLevel::classForSize(size);
-    }
-
-private:
-    NextLevel* m_ptr;
-};
-
-template<class NextLevel, unsigned log2Entries>
-class AllocationTableDirectory {
-    typedef uint64_t BitField;
-
-public:
-    static const unsigned log2SubregionSize = NextLevel::log2RegionSize;
-    static const unsigned log2RegionSize = log2SubregionSize + log2Entries;
-
-    static const size_t subregionSize = TwoPow(log2SubregionSize);
-    static const size_t regionSize = TwoPow(log2RegionSize);
-    static const unsigned entries = TwoPow(log2Entries);
-    COMPILE_ASSERT(entries <= (sizeof(BitField) * 8), AllocationTableDirectory_entries_fit_in_BitField);
-
-    AllocationTableDirectory()
-        : m_full(0)
-        , m_hasSuballocation(0)
-    {

     }
     }

-
-    ~AllocationTableDirectory()
-    {
-        ASSERT(isEmpty());
-    }
-
-    size_t allocate(AllocationTableSizeClass& sizeClass)
-    {
-        ASSERT(sizeClass.blockSize() <= subregionSize);
-        ASSERT(!isFull());
-
-        if (sizeClass.blockSize() < subregionSize) {
-            BitField bit = 1;
-            for (unsigned i = 0; i < entries; ++i, bit += bit) {
-                if (m_full & bit)
-                    continue;
-                size_t location = m_suballocations[i].allocate(sizeClass);
-                if (location != notFound) {
-                    // If this didn't already have a subregion, it does now!
-                    m_hasSuballocation |= bit;
-                    // Mirror the suballocation's full bit.
-                    if (m_suballocations[i].isFull())
-                        m_full |= bit;
-                    return (i * subregionSize) + location;
-                }
-            }
-            return notFound;
-        }
-
-        // A block is allocated if either it is fully allocated or contains suballocations.
-        BitField allocated = m_full | m_hasSuballocation;
-
-        size_t alignment = sizeClass.blockAlignment();
-        size_t count = sizeClass.blockCount();
-        // Use this mask to check for spans of free blocks.
-        BitField mask = ((1ull << count) - 1) << (alignment - count);
-
-        // Step in units of alignment size.
-        for (unsigned i = 0; i < entries; i += alignment) {
-            if (!(allocated & mask)) {
-                m_full |= mask;
-                return (i + (alignment - count)) << log2SubregionSize;
+    
+    virtual void notifyPageIsFree(void* page) override
+    {
+#if USE(MADV_FREE_FOR_JIT_MEMORY)
+        for (;;) {
+            int result = madvise(page, pageSize(), MADV_FREE);
+            if (!result)
+                return;
+            ASSERT(result == -1);
+            if (errno != EAGAIN) {
+                RELEASE_ASSERT_NOT_REACHED(); // In debug mode, this should be a hard failure.
+                break; // In release mode, we should just ignore the error - not returning memory to the OS is better than crashing, especially since we _will_ be able to reuse the memory internally anyway.

             }
             }

-            mask <<= alignment;
-        }
-        return notFound;
-    }
-
-    void free(size_t location, AllocationTableSizeClass& sizeClass)
-    {
-        ASSERT(sizeClass.blockSize() <= subregionSize);
-
-        size_t entry = location >> log2SubregionSize;
-
-        if (sizeClass.blockSize() < subregionSize) {
-            BitField bit = 1ull << entry;
-            m_suballocations[entry].free(location & (subregionSize - 1), sizeClass);
-            // Check if the suballocation is now empty.
-            if (m_suballocations[entry].isEmpty())
-                m_hasSuballocation &= ~bit;
-            // No need to check, it clearly isn't full any more!
-            m_full &= ~bit;
-        } else {
-            size_t count = sizeClass.blockCount();
-            BitField mask = ((1ull << count) - 1) << entry;
-            ASSERT((m_full & mask) == mask);
-            ASSERT(!(m_hasSuballocation & mask));
-            m_full &= ~mask;
-        }
-    }
-
-    bool isEmpty()
-    {
-        return !(m_full | m_hasSuballocation);
-    }
-
-    bool isFull()
-    {   
-        return !~m_full;
-    }
-
-    static size_t size()
-    {
-        return regionSize;
-    }
-
-    static AllocationTableSizeClass classForSize(size_t size)
-    {
-        if (size < subregionSize) {
-            AllocationTableSizeClass sizeClass = NextLevel::classForSize(size);
-            if (sizeClass.size() < NextLevel::size())
-                return sizeClass;

         }
         }

-        return AllocationTableSizeClass(size, subregionSize, log2SubregionSize);
-    }
-
-#ifndef NDEBUG
-    void dump(size_t parentOffset = 0, unsigned indent = 0)
-    {
-        for (unsigned i = 0; i < indent; ++i)
-            fprintf(stderr, "    ");
-        fprintf(stderr, "%08x: [", (int)parentOffset);
-        for (unsigned i = 0; i < entries; ++i) {
-            BitField bit = 1ull << i;
-            char c = m_hasSuballocation & bit
-                ? (m_full & bit ? 'N' : 'n')
-                : (m_full & bit ? 'F' : '-');
-            fprintf(stderr, "%c", c);
-        }
-        fprintf(stderr, "]\n");
-
-        for (unsigned i = 0; i < entries; ++i) {
-            BitField bit = 1ull << i;
-            size_t offset = parentOffset | (subregionSize * i);
-            if (m_hasSuballocation & bit)
-                m_suballocations[i].dump(offset, indent + 1);
-        }
-    }
-#endif
-
-private:
-    NextLevel m_suballocations[entries];
-    // Subregions exist in one of four states:
-    // (1) empty (both bits clear)
-    // (2) fully allocated as a single allocation (m_full set)
-    // (3) partially allocated through suballocations (m_hasSuballocation set)
-    // (4) fully allocated through suballocations (both bits set)
-    BitField m_full;
-    BitField m_hasSuballocation;
-};
-
-typedef AllocationTableLeaf<6> PageTables256KB;
-typedef AllocationTableDirectory<PageTables256KB, 6> PageTables16MB;
-typedef AllocationTableDirectory<LazyAllocationTable<PageTables16MB>, 1> PageTables32MB;
-typedef AllocationTableDirectory<LazyAllocationTable<PageTables16MB>, 6> PageTables1GB;
-
-#if CPU(ARM)
-typedef PageTables16MB FixedVMPoolPageTables;
-#elif CPU(X86_64) && !OS(LINUX)
-typedef PageTables1GB FixedVMPoolPageTables;
-#else
-typedef PageTables32MB FixedVMPoolPageTables;
-#endif
-
-class FixedVMPoolAllocator
-{
-public:
-    FixedVMPoolAllocator()
-    {
-        m_base = mmap(0, FixedVMPoolPageTables::size(), INITIAL_PROTECTION_FLAGS, MMAP_FLAGS, VM_TAG_FOR_EXECUTABLEALLOCATOR_MEMORY, 0);
-
-        if (m_base == MAP_FAILED) {
-#if ENABLE(INTERPRETER)
-            m_base = 0;

 #else
 #else

-            CRASH();
+        m_reservation.decommit(page, pageSize());

 #endif
 #endif

-        } else {
-            // For simplicity, we keep all memory in m_freeList in a 'released' state.
-            // This means that we can simply reuse all memory when allocating, without
-            // worrying about it's previous state, and also makes coalescing m_freeList
-            // simpler since we need not worry about the possibility of coalescing released
-            // chunks with non-released ones.
-            release(m_base, FixedVMPoolPageTables::size());
-        }
-    }
- 
-    void* alloc(size_t size)
-    {
-        ASSERT(size);
-        AllocationTableSizeClass sizeClass = classForSize(size);
-        ASSERT(sizeClass.size());
-        if (sizeClass.size() >= FixedVMPoolPageTables::size())
-            CRASH();
-
-        if (m_pages.isFull())
-            CRASH();
-        size_t offset = m_pages.allocate(sizeClass);
-        if (offset == notFound)
-            CRASH();
-
-        void* result = offsetToPointer(offset);
-        reuse(result, size);
-        return result;
-    }
-
-    void free(void* pointer, size_t size)
-    {
-        release(pointer, size);
-
-        ASSERT(size);
-        AllocationTableSizeClass sizeClass = classForSize(size);
-        ASSERT(sizeClass.size());
-        ASSERT(sizeClass.size() < FixedVMPoolPageTables::size());
-
-        m_pages.free(pointerToOffset(pointer), sizeClass);
-    }
-
-    bool isValid() const
-    {
-        return !!m_base;

     }
 
 private:
     }
 
 private:

-    // Use madvise as apropriate to prevent freed pages from being spilled,
-    // and to attempt to ensure that used memory is reported correctly.
-#if HAVE(MADV_FREE_REUSE)
-    void release(void* position, size_t size)
-    {
-        while (madvise(position, size, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) { }
-    }
-
-    void reuse(void* position, size_t size)
-    {
-        while (madvise(position, size, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { }
-    }
-#elif HAVE(MADV_FREE)
-    void release(void* position, size_t size)
-    {
-        while (madvise(position, size, MADV_FREE) == -1 && errno == EAGAIN) { }
-    }
-    
-    void reuse(void*, size_t) {}
-#elif HAVE(MADV_DONTNEED)
-    void release(void* position, size_t size)
-    {
-        while (madvise(position, size, MADV_DONTNEED) == -1 && errno == EAGAIN) { }
-    }
-
-    void reuse(void*, size_t) {}
-#else
-    void release(void*, size_t) {}
-    void reuse(void*, size_t) {}
-#endif
-
-    AllocationTableSizeClass classForSize(size_t size)
-    {
-        return FixedVMPoolPageTables::classForSize(size);
-    }
+    PageReservation m_reservation;
+};

 
 

-    void* offsetToPointer(size_t offset)
-    {
-        return reinterpret_cast<void*>(reinterpret_cast<intptr_t>(m_base) + offset);
-    }
+static FixedVMPoolExecutableAllocator* allocator;

 
 

-    size_t pointerToOffset(void* pointer)
-    {
-        return reinterpret_cast<intptr_t>(pointer) - reinterpret_cast<intptr_t>(m_base);
-    }
+void ExecutableAllocator::initializeAllocator()
+{
+    ASSERT(!allocator);
+    allocator = new FixedVMPoolExecutableAllocator();
+    CodeProfiling::notifyAllocator(allocator);
+}

 
 

-    void* m_base;
-    FixedVMPoolPageTables m_pages;
-};
+ExecutableAllocator::ExecutableAllocator(VM&)
+{
+    ASSERT(allocator);
+}

 
 

-void ExecutableAllocator::intializePageSize()
+ExecutableAllocator::~ExecutableAllocator()

 {
 {

-    ExecutableAllocator::pageSize = getpagesize();

 }
 
 }
 

-static FixedVMPoolAllocator* allocator = 0;
-static size_t allocatedCount = 0;
-static SpinLock spinlock = SPINLOCK_INITIALIZER;
+FixedVMPoolExecutableAllocator::~FixedVMPoolExecutableAllocator()
+{
+    m_reservation.deallocate();
+}

 
 bool ExecutableAllocator::isValid() const
 {
 
 bool ExecutableAllocator::isValid() const
 {

-    SpinLockHolder lock_holder(&spinlock);
-    if (!allocator)
-        allocator = new FixedVMPoolAllocator();
-    return allocator->isValid();
+    return !!allocator->bytesReserved();

 }
 
 }
 

-ExecutablePool::Allocation ExecutablePool::systemAlloc(size_t size)
+bool ExecutableAllocator::underMemoryPressure()

 {
 {

-    SpinLockHolder lock_holder(&spinlock);
+    MetaAllocator::Statistics statistics = allocator->currentStatistics();
+    return statistics.bytesAllocated > statistics.bytesReserved / 2;
+}

 
 

-    if (!allocator)
-        allocator = new FixedVMPoolAllocator();
-    ExecutablePool::Allocation alloc = {reinterpret_cast<char*>(allocator->alloc(size)), size};
-    allocatedCount += size;
-    return alloc;
+double ExecutableAllocator::memoryPressureMultiplier(size_t addedMemoryUsage)
+{
+    MetaAllocator::Statistics statistics = allocator->currentStatistics();
+    ASSERT(statistics.bytesAllocated <= statistics.bytesReserved);
+    size_t bytesAllocated = statistics.bytesAllocated + addedMemoryUsage;
+    if (bytesAllocated >= statistics.bytesReserved)
+        bytesAllocated = statistics.bytesReserved;
+    double result = 1.0;
+    size_t divisor = statistics.bytesReserved - bytesAllocated;
+    if (divisor)
+        result = static_cast<double>(statistics.bytesReserved) / divisor;
+    if (result < 1.0)
+        result = 1.0;
+    return result;

 }
 
 }
 

-void ExecutablePool::systemRelease(const ExecutablePool::Allocation& allocation) 
+PassRefPtr<ExecutableMemoryHandle> ExecutableAllocator::allocate(VM& vm, size_t sizeInBytes, void* ownerUID, JITCompilationEffort effort)

 {
 {

-    SpinLockHolder lock_holder(&spinlock);
+    RefPtr<ExecutableMemoryHandle> result = allocator->allocate(sizeInBytes, ownerUID);
+    if (!result) {
+        if (effort == JITCompilationCanFail)
+            return result;
+        releaseExecutableMemory(vm);
+        result = allocator->allocate(sizeInBytes, ownerUID);
+        RELEASE_ASSERT(result);
+    }
+    return result.release();
+}

 
 

-    ASSERT(allocator);
-    allocator->free(allocation.pages, allocation.size);
-    allocatedCount -= allocation.size;
+size_t ExecutableAllocator::committedByteCount()
+{
+    return allocator->bytesCommitted();

 }
 
 }
 

-bool ExecutablePool::underMemoryPressure()
+#if ENABLE(META_ALLOCATOR_PROFILE)
+void ExecutableAllocator::dumpProfile()

 {
 {

-    // Technically we should take the spin lock here, but we don't
-    // care if we get stale data.  This is only really a heuristic
-    // anyway.
-    return allocatedCount > (FixedVMPoolPageTables::size() / 2);
+    allocator->dumpProfile();

 }
 }

+#endif

 
 }
 
 
 }
 

-#endif // HAVE(ASSEMBLER)
+
+#endif // ENABLE(EXECUTABLE_ALLOCATOR_FIXED)