#include "config.h"
#include "CopiedSpace.h"
-#include "CopiedSpaceInlineMethods.h"
+#include "CopiedSpaceInlines.h"
#include "GCActivityCallback.h"
+#include "JSCInlines.h"
+#include "Options.h"
namespace JSC {
CopiedSpace::CopiedSpace(Heap* heap)
: m_heap(heap)
- , m_toSpace(0)
- , m_fromSpace(0)
, m_inCopyingPhase(false)
+ , m_shouldDoCopyPhase(false)
, m_numberOfLoanedBlocks(0)
+ , m_bytesRemovedFromOldSpaceDueToReallocation(0)
{
}
+CopiedSpace::~CopiedSpace()
+{
+ while (!m_oldGen.toSpace->isEmpty())
+ CopiedBlock::destroy(m_oldGen.toSpace->removeHead());
+
+ while (!m_oldGen.fromSpace->isEmpty())
+ CopiedBlock::destroy(m_oldGen.fromSpace->removeHead());
+
+ while (!m_oldGen.oversizeBlocks.isEmpty())
+ CopiedBlock::destroy(m_oldGen.oversizeBlocks.removeHead());
+
+ while (!m_newGen.toSpace->isEmpty())
+ CopiedBlock::destroy(m_newGen.toSpace->removeHead());
+
+ while (!m_newGen.fromSpace->isEmpty())
+ CopiedBlock::destroy(m_newGen.fromSpace->removeHead());
+
+ while (!m_newGen.oversizeBlocks.isEmpty())
+ CopiedBlock::destroy(m_newGen.oversizeBlocks.removeHead());
+
+ ASSERT(m_oldGen.toSpace->isEmpty());
+ ASSERT(m_oldGen.fromSpace->isEmpty());
+ ASSERT(m_oldGen.oversizeBlocks.isEmpty());
+ ASSERT(m_newGen.toSpace->isEmpty());
+ ASSERT(m_newGen.fromSpace->isEmpty());
+ ASSERT(m_newGen.oversizeBlocks.isEmpty());
+}
+
void CopiedSpace::init()
{
- m_toSpace = &m_blocks1;
- m_fromSpace = &m_blocks2;
+ m_oldGen.toSpace = &m_oldGen.blocks1;
+ m_oldGen.fromSpace = &m_oldGen.blocks2;
- if (!addNewBlock())
- CRASH();
+ m_newGen.toSpace = &m_newGen.blocks1;
+ m_newGen.fromSpace = &m_newGen.blocks2;
+
+ allocateBlock();
}
CheckedBoolean CopiedSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
if (isOversize(bytes))
return tryAllocateOversize(bytes, outPtr);
- ASSERT(m_heap->globalData()->apiLock().currentThreadIsHoldingLock());
+ ASSERT(m_heap->vm()->currentThreadIsHoldingAPILock());
m_heap->didAllocate(m_allocator.currentCapacity());
- if (!addNewBlock()) {
- *outPtr = 0;
- return false;
- }
- *outPtr = m_allocator.allocate(bytes);
- ASSERT(*outPtr);
+ allocateBlock();
+
+ *outPtr = m_allocator.forceAllocate(bytes);
return true;
}
{
ASSERT(isOversize(bytes));
- size_t blockSize = WTF::roundUpToMultipleOf(WTF::pageSize(), sizeof(CopiedBlock) + bytes);
-
- PageAllocationAligned allocation = PageAllocationAligned::allocate(blockSize, WTF::pageSize(), OSAllocator::JSGCHeapPages);
- if (!static_cast<bool>(allocation)) {
- *outPtr = 0;
- return false;
- }
-
- CopiedBlock* block = new (NotNull, allocation.base()) CopiedBlock(allocation);
- m_oversizeBlocks.push(block);
- m_oversizeFilter.add(reinterpret_cast<Bits>(block));
+ CopiedBlock* block = CopiedBlock::create(WTF::roundUpToMultipleOf<sizeof(double)>(sizeof(CopiedBlock) + bytes));
+ m_newGen.oversizeBlocks.push(block);
+ m_newGen.blockFilter.add(reinterpret_cast<Bits>(block));
+ m_blockSet.add(block);
+ ASSERT(!block->isOld());
- *outPtr = allocateFromBlock(block, bytes);
+ CopiedAllocator allocator;
+ allocator.setCurrentBlock(block);
+ *outPtr = allocator.forceAllocate(bytes);
+ allocator.resetCurrentBlock();
- m_heap->didAllocate(blockSize);
+ m_heap->didAllocate(block->capacity());
return true;
}
return true;
void* oldPtr = *ptr;
- ASSERT(!m_heap->globalData()->isInitializingObject());
-
- if (isOversize(oldSize) || isOversize(newSize))
+ ASSERT(!m_heap->vm()->isInitializingObject());
+
+ if (CopiedSpace::blockFor(oldPtr)->isOversize() || isOversize(newSize))
return tryReallocateOversize(ptr, oldSize, newSize);
-
- if (m_allocator.wasLastAllocation(oldPtr, oldSize)) {
- size_t delta = newSize - oldSize;
- if (m_allocator.fitsInCurrentBlock(delta)) {
- (void)m_allocator.allocate(delta);
- return true;
- }
- }
+
+ if (m_allocator.tryReallocate(oldPtr, oldSize, newSize))
+ return true;
void* result = 0;
if (!tryAllocate(newSize, &result)) {
memcpy(newPtr, oldPtr, oldSize);
- if (isOversize(oldSize)) {
- CopiedBlock* oldBlock = oversizeBlockFor(oldPtr);
- m_oversizeBlocks.remove(oldBlock);
- oldBlock->m_allocation.deallocate();
+ CopiedBlock* oldBlock = CopiedSpace::blockFor(oldPtr);
+ if (oldBlock->isOversize()) {
+ // FIXME: Eagerly deallocating the old space block probably buys more confusion than
+ // value.
+ // https://bugs.webkit.org/show_bug.cgi?id=144750
+ if (oldBlock->isOld()) {
+ m_bytesRemovedFromOldSpaceDueToReallocation += oldBlock->size();
+ m_oldGen.oversizeBlocks.remove(oldBlock);
+ } else
+ m_newGen.oversizeBlocks.remove(oldBlock);
+ m_blockSet.remove(oldBlock);
+ CopiedBlock::destroy(oldBlock);
}
*ptr = newPtr;
return true;
}
-void CopiedSpace::doneFillingBlock(CopiedBlock* block)
+void CopiedSpace::doneFillingBlock(CopiedBlock* block, CopiedBlock** exchange)
{
- ASSERT(block);
- ASSERT(block->m_offset < reinterpret_cast<char*>(block) + HeapBlock::s_blockSize);
ASSERT(m_inCopyingPhase);
+
+ if (exchange)
+ *exchange = allocateBlockForCopyingPhase();
+
+ if (!block)
+ return;
- if (block->m_offset == block->payload()) {
- recycleBlock(block);
+ if (!block->dataSize()) {
+ recycleBorrowedBlock(block);
return;
}
+ block->zeroFillWilderness();
+
{
- MutexLocker locker(m_toSpaceLock);
- m_toSpace->push(block);
- m_toSpaceSet.add(block);
- m_toSpaceFilter.add(reinterpret_cast<Bits>(block));
+ // Always put the block into the old gen because it's being promoted!
+ SpinLockHolder locker(&m_toSpaceLock);
+ m_oldGen.toSpace->push(block);
+ m_blockSet.add(block);
+ m_oldGen.blockFilter.add(reinterpret_cast<Bits>(block));
}
{
MutexLocker locker(m_loanedBlocksLock);
ASSERT(m_numberOfLoanedBlocks > 0);
+ ASSERT(m_inCopyingPhase);
m_numberOfLoanedBlocks--;
if (!m_numberOfLoanedBlocks)
m_loanedBlocksCondition.signal();
}
}
+void CopiedSpace::didStartFullCollection()
+{
+ ASSERT(heap()->operationInProgress() == FullCollection);
+ ASSERT(m_oldGen.fromSpace->isEmpty());
+ ASSERT(m_newGen.fromSpace->isEmpty());
+
+#ifndef NDEBUG
+ for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+ ASSERT(!block->liveBytes());
+
+ for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+ ASSERT(!block->liveBytes());
+#endif
+
+ for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+ block->didSurviveGC();
+
+ for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+ block->didSurviveGC();
+}
+
void CopiedSpace::doneCopying()
{
{
m_loanedBlocksCondition.wait(m_loanedBlocksLock);
}
- ASSERT(m_inCopyingPhase);
+ ASSERT(m_inCopyingPhase == m_shouldDoCopyPhase);
m_inCopyingPhase = false;
- while (!m_fromSpace->isEmpty()) {
- CopiedBlock* block = static_cast<CopiedBlock*>(m_fromSpace->removeHead());
- if (block->m_isPinned) {
- block->m_isPinned = false;
- // We don't add the block to the toSpaceSet because it was never removed.
- ASSERT(m_toSpaceSet.contains(block));
- m_toSpaceFilter.add(reinterpret_cast<Bits>(block));
- m_toSpace->push(block);
- continue;
- }
- m_toSpaceSet.remove(block);
- m_heap->blockAllocator().deallocate(block);
+ DoublyLinkedList<CopiedBlock>* toSpace;
+ DoublyLinkedList<CopiedBlock>* fromSpace;
+ TinyBloomFilter* blockFilter;
+ if (heap()->operationInProgress() == FullCollection) {
+ toSpace = m_oldGen.toSpace;
+ fromSpace = m_oldGen.fromSpace;
+ blockFilter = &m_oldGen.blockFilter;
+ } else {
+ toSpace = m_newGen.toSpace;
+ fromSpace = m_newGen.fromSpace;
+ blockFilter = &m_newGen.blockFilter;
}
- CopiedBlock* curr = static_cast<CopiedBlock*>(m_oversizeBlocks.head());
- while (curr) {
- CopiedBlock* next = static_cast<CopiedBlock*>(curr->next());
- if (!curr->m_isPinned) {
- m_oversizeBlocks.remove(curr);
- curr->m_allocation.deallocate();
- } else
- curr->m_isPinned = false;
- curr = next;
+ while (!fromSpace->isEmpty()) {
+ CopiedBlock* block = fromSpace->removeHead();
+ // We don't add the block to the blockSet because it was never removed.
+ ASSERT(m_blockSet.contains(block));
+ blockFilter->add(reinterpret_cast<Bits>(block));
+ block->didSurviveGC();
+ toSpace->push(block);
}
- if (!m_toSpace->head()) {
- if (!addNewBlock())
- CRASH();
- } else
- m_allocator.resetCurrentBlock(static_cast<CopiedBlock*>(m_toSpace->head()));
-}
-
-CheckedBoolean CopiedSpace::getFreshBlock(AllocationEffort allocationEffort, CopiedBlock** outBlock)
-{
- CopiedBlock* block = 0;
- if (allocationEffort == AllocationMustSucceed) {
- if (HeapBlock* heapBlock = m_heap->blockAllocator().allocate())
- block = new (NotNull, heapBlock) CopiedBlock(heapBlock->m_allocation);
- else if (!allocateNewBlock(&block)) {
- *outBlock = 0;
- ASSERT_NOT_REACHED();
- return false;
- }
- } else {
- ASSERT(allocationEffort == AllocationCanFail);
- if (m_heap->shouldCollect())
- m_heap->collect(Heap::DoNotSweep);
-
- if (!getFreshBlock(AllocationMustSucceed, &block)) {
- *outBlock = 0;
- ASSERT_NOT_REACHED();
- return false;
- }
+ if (heap()->operationInProgress() == EdenCollection) {
+ m_oldGen.toSpace->append(*m_newGen.toSpace);
+ m_oldGen.oversizeBlocks.append(m_newGen.oversizeBlocks);
+ m_oldGen.blockFilter.add(m_newGen.blockFilter);
+ m_newGen.blockFilter.reset();
}
- ASSERT(block);
- ASSERT(is8ByteAligned(block->m_offset));
- *outBlock = block;
- return true;
-}
-void CopiedSpace::freeAllBlocks()
-{
- while (!m_toSpace->isEmpty())
- m_heap->blockAllocator().deallocate(m_toSpace->removeHead());
+ ASSERT(m_newGen.toSpace->isEmpty());
+ ASSERT(m_newGen.fromSpace->isEmpty());
+ ASSERT(m_newGen.oversizeBlocks.isEmpty());
- while (!m_fromSpace->isEmpty())
- m_heap->blockAllocator().deallocate(m_fromSpace->removeHead());
+ allocateBlock();
- while (!m_oversizeBlocks.isEmpty())
- m_oversizeBlocks.removeHead()->m_allocation.deallocate();
+ m_shouldDoCopyPhase = false;
}
size_t CopiedSpace::size()
{
size_t calculatedSize = 0;
- for (CopiedBlock* block = static_cast<CopiedBlock*>(m_toSpace->head()); block; block = static_cast<CopiedBlock*>(block->next()))
+ for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+ calculatedSize += block->size();
+
+ for (CopiedBlock* block = m_oldGen.fromSpace->head(); block; block = block->next())
+ calculatedSize += block->size();
+
+ for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
calculatedSize += block->size();
- for (CopiedBlock* block = static_cast<CopiedBlock*>(m_fromSpace->head()); block; block = static_cast<CopiedBlock*>(block->next()))
+ for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
calculatedSize += block->size();
- for (CopiedBlock* block = static_cast<CopiedBlock*>(m_oversizeBlocks.head()); block; block = static_cast<CopiedBlock*>(block->next()))
+ for (CopiedBlock* block = m_newGen.fromSpace->head(); block; block = block->next())
+ calculatedSize += block->size();
+
+ for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
calculatedSize += block->size();
return calculatedSize;
{
size_t calculatedCapacity = 0;
- for (CopiedBlock* block = static_cast<CopiedBlock*>(m_toSpace->head()); block; block = static_cast<CopiedBlock*>(block->next()))
+ for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+ calculatedCapacity += block->capacity();
+
+ for (CopiedBlock* block = m_oldGen.fromSpace->head(); block; block = block->next())
+ calculatedCapacity += block->capacity();
+
+ for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+ calculatedCapacity += block->capacity();
+
+ for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
calculatedCapacity += block->capacity();
- for (CopiedBlock* block = static_cast<CopiedBlock*>(m_fromSpace->head()); block; block = static_cast<CopiedBlock*>(block->next()))
+ for (CopiedBlock* block = m_newGen.fromSpace->head(); block; block = block->next())
calculatedCapacity += block->capacity();
- for (CopiedBlock* block = static_cast<CopiedBlock*>(m_oversizeBlocks.head()); block; block = static_cast<CopiedBlock*>(block->next()))
+ for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
calculatedCapacity += block->capacity();
return calculatedCapacity;
}
-static bool isBlockListPagedOut(double deadline, DoublyLinkedList<HeapBlock>* list)
+static bool isBlockListPagedOut(double deadline, DoublyLinkedList<CopiedBlock>* list)
{
unsigned itersSinceLastTimeCheck = 0;
- HeapBlock* current = list->head();
+ CopiedBlock* current = list->head();
while (current) {
current = current->next();
++itersSinceLastTimeCheck;
bool CopiedSpace::isPagedOut(double deadline)
{
- return isBlockListPagedOut(deadline, m_toSpace)
- || isBlockListPagedOut(deadline, m_fromSpace)
- || isBlockListPagedOut(deadline, &m_oversizeBlocks);
+ return isBlockListPagedOut(deadline, m_oldGen.toSpace)
+ || isBlockListPagedOut(deadline, m_oldGen.fromSpace)
+ || isBlockListPagedOut(deadline, &m_oldGen.oversizeBlocks)
+ || isBlockListPagedOut(deadline, m_newGen.toSpace)
+ || isBlockListPagedOut(deadline, m_newGen.fromSpace)
+ || isBlockListPagedOut(deadline, &m_newGen.oversizeBlocks);
}
} // namespace JSC
projects / apple / javascriptcore.git / blobdiff