JavaScriptCore-7600.1.4.15.12.tar.gz

[apple/javascriptcore.git] / dfg / DFGJITCompiler.cpp

diff --git a/dfg/DFGJITCompiler.cpp b/dfg/DFGJITCompiler.cpp
index 9e946d22a45b7fef179746de58c1c75c570c0469..c54746e6ba98b38d8a7cad2fedff45c32b658441 100644 (file)
--- a/dfg/DFGJITCompiler.cpp
+++ b/dfg/DFGJITCompiler.cpp
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (C) 2011 Apple Inc. All rights reserved.
+ * Copyright (C) 2011, 2013, 2014 Apple Inc. All rights reserved.

  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions


@@ -28,425 +28,437 @@
 
 #if ENABLE(DFG_JIT)
 
 
 #if ENABLE(DFG_JIT)
 

+#include "ArityCheckFailReturnThunks.h"

 #include "CodeBlock.h"
 #include "CodeBlock.h"

-#include "DFGJITCodeGenerator.h"
-#include "DFGNonSpeculativeJIT.h"
+#include "DFGFailedFinalizer.h"
+#include "DFGInlineCacheWrapperInlines.h"
+#include "DFGJITCode.h"
+#include "DFGJITFinalizer.h"
+#include "DFGOSRExitCompiler.h"

 #include "DFGOperations.h"
 #include "DFGRegisterBank.h"
 #include "DFGOperations.h"
 #include "DFGRegisterBank.h"

+#include "DFGSlowPathGenerator.h"

 #include "DFGSpeculativeJIT.h"
 #include "DFGSpeculativeJIT.h"

-#include "JSGlobalData.h"
+#include "DFGThunks.h"
+#include "JSCJSValueInlines.h"

 #include "LinkBuffer.h"
 #include "LinkBuffer.h"

+#include "MaxFrameExtentForSlowPathCall.h"
+#include "JSCInlines.h"
+#include "VM.h"

 
 namespace JSC { namespace DFG {
 
 
 namespace JSC { namespace DFG {
 

-// This method used to fill a numeric value to a FPR when linking speculative -> non-speculative.
-void JITCompiler::fillNumericToDouble(NodeIndex nodeIndex, FPRReg fpr, GPRReg temporary)
+JITCompiler::JITCompiler(Graph& dfg)
+    : CCallHelpers(&dfg.m_vm, dfg.m_codeBlock)
+    , m_graph(dfg)
+    , m_jitCode(adoptRef(new JITCode()))
+    , m_blockHeads(dfg.numBlocks())

 {
 {

-    Node& node = graph()[nodeIndex];
-
-    if (node.isConstant()) {
-        ASSERT(node.op == DoubleConstant);
-        move(MacroAssembler::ImmPtr(reinterpret_cast<void*>(reinterpretDoubleToIntptr(valueOfDoubleConstant(nodeIndex)))), temporary);
-        movePtrToDouble(temporary, fpr);
-    } else {
-        loadPtr(addressFor(node.virtualRegister()), temporary);
-        Jump isInteger = branchPtr(MacroAssembler::AboveOrEqual, temporary, GPRInfo::tagTypeNumberRegister);
-        jitAssertIsJSDouble(temporary);
-        addPtr(GPRInfo::tagTypeNumberRegister, temporary);
-        movePtrToDouble(temporary, fpr);
-        Jump hasUnboxedDouble = jump();
-        isInteger.link(this);
-        convertInt32ToDouble(temporary, fpr);
-        hasUnboxedDouble.link(this);
-    }
+    if (shouldShowDisassembly() || m_graph.m_vm.m_perBytecodeProfiler)
+        m_disassembler = adoptPtr(new Disassembler(dfg));

 }
 
 }
 

-// This method used to fill an integer value to a GPR when linking speculative -> non-speculative.
-void JITCompiler::fillInt32ToInteger(NodeIndex nodeIndex, GPRReg gpr)
+JITCompiler::~JITCompiler()

 {
 {

-    Node& node = graph()[nodeIndex];
-
-    if (node.isConstant()) {
-        ASSERT(node.op == Int32Constant);
-        move(MacroAssembler::Imm32(valueOfInt32Constant(nodeIndex)), gpr);
-    } else {
-#if DFG_JIT_ASSERT
-        // Redundant load, just so we can check the tag!
-        loadPtr(addressFor(node.virtualRegister()), gpr);
-        jitAssertIsJSInt32(gpr);
-#endif
-        load32(addressFor(node.virtualRegister()), gpr);
-    }

 }
 
 }
 

-// This method used to fill a JSValue to a GPR when linking speculative -> non-speculative.
-void JITCompiler::fillToJS(NodeIndex nodeIndex, GPRReg gpr)
+void JITCompiler::linkOSRExits()

 {
 {

-    Node& node = graph()[nodeIndex];
-
-    if (node.isConstant()) {
-        if (isInt32Constant(nodeIndex)) {
-            JSValue jsValue = jsNumber(valueOfInt32Constant(nodeIndex));
-            move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gpr);
-        } else if (isDoubleConstant(nodeIndex)) {
-            JSValue jsValue(JSValue::EncodeAsDouble, valueOfDoubleConstant(nodeIndex));
-            move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gpr);
-        } else {
-            ASSERT(isJSConstant(nodeIndex));
-            JSValue jsValue = valueOfJSConstant(nodeIndex);
-            move(MacroAssembler::ImmPtr(JSValue::encode(jsValue)), gpr);
+    ASSERT(m_jitCode->osrExit.size() == m_exitCompilationInfo.size());
+    if (m_graph.compilation()) {
+        for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) {
+            OSRExitCompilationInfo& info = m_exitCompilationInfo[i];
+            Vector<Label> labels;
+            if (!info.m_failureJumps.empty()) {
+                for (unsigned j = 0; j < info.m_failureJumps.jumps().size(); ++j)
+                    labels.append(info.m_failureJumps.jumps()[j].label());
+            } else
+                labels.append(info.m_replacementSource);
+            m_exitSiteLabels.append(labels);

         }
         }

-        return;

     }
     }

+    
+    for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) {
+        OSRExit& exit = m_jitCode->osrExit[i];
+        OSRExitCompilationInfo& info = m_exitCompilationInfo[i];
+        JumpList& failureJumps = info.m_failureJumps;
+        if (!failureJumps.empty())
+            failureJumps.link(this);
+        else
+            info.m_replacementDestination = label();
+        jitAssertHasValidCallFrame();
+        store32(TrustedImm32(i), &vm()->osrExitIndex);
+        exit.setPatchableCodeOffset(patchableJump());
+    }
+}

 
 

-    loadPtr(addressFor(node.virtualRegister()), gpr);
+void JITCompiler::compileEntry()
+{
+    // This code currently matches the old JIT. In the function header we need to
+    // save return address and call frame via the prologue and perform a fast stack check.
+    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=56292
+    // We'll need to convert the remaining cti_ style calls (specifically the stack
+    // check) which will be dependent on stack layout. (We'd need to account for this in
+    // both normal return code and when jumping to an exception handler).
+    emitFunctionPrologue();
+    emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
+    jitAssertTagsInPlace();

 }
 
 }
 

-void JITCompiler::jumpFromSpeculativeToNonSpeculative(const SpeculationCheck& check, const EntryLocation& entry, SpeculationRecovery* recovery)
+void JITCompiler::compileBody()

 {
 {

-    ASSERT(check.m_nodeIndex == entry.m_nodeIndex);
-
-    // Link the jump from the Speculative path to here.
-    check.m_check.link(this);
-
-    // Does this speculation check require any additional recovery to be performed,
-    // to restore any state that has been overwritten before we enter back in to the
-    // non-speculative path.
-    if (recovery) {
-        // The only additional recovery we currently support is for integer add operation
-        ASSERT(recovery->type() == SpeculativeAdd);
-        // Revert the add.
-        sub32(recovery->src(), recovery->dest());
-    }
+    // We generate the speculative code path, followed by OSR exit code to return
+    // to the old JIT code if speculations fail.

 
 

-    // FIXME: - This is hideously inefficient!
-    // Where a value is live in a register in the speculative path, and is required in a register
-    // on the non-speculative path, we should not need to be spilling it and reloading (we may
-    // need to spill anyway, if the value is marked as spilled on the non-speculative path).
-    // This may also be spilling values that don't need spilling, e.g. are already spilled,
-    // are constants, or are arguments.
-
-    // Spill all GPRs in use by the speculative path.
-    for (unsigned index = 0; index < GPRInfo::numberOfRegisters; ++index) {
-        NodeIndex nodeIndex = check.m_gprInfo[index].nodeIndex;
-        if (nodeIndex == NoNode)
-            continue;
+    bool compiledSpeculative = m_speculative->compile();
+    ASSERT_UNUSED(compiledSpeculative, compiledSpeculative);
+}

 
 

-        DataFormat dataFormat = check.m_gprInfo[index].format;
-        VirtualRegister virtualRegister = graph()[nodeIndex].virtualRegister();
+void JITCompiler::compileExceptionHandlers()
+{
+    if (m_exceptionChecks.empty() && m_exceptionChecksWithCallFrameRollback.empty())
+        return;

 
 

-        ASSERT(dataFormat == DataFormatInteger || DataFormatCell || dataFormat & DataFormatJS);
-        if (dataFormat == DataFormatInteger)
-            orPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::toRegister(index));
-        storePtr(GPRInfo::toRegister(index), addressFor(virtualRegister));
+    Jump doLookup;
+
+    if (!m_exceptionChecksWithCallFrameRollback.empty()) {
+        m_exceptionChecksWithCallFrameRollback.link(this);
+        emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::argumentGPR1);
+        doLookup = jump();

     }
 
     }
 

-    // Spill all FPRs in use by the speculative path.
-    for (unsigned index = 0; index < FPRInfo::numberOfRegisters; ++index) {
-        NodeIndex nodeIndex = check.m_fprInfo[index];
-        if (nodeIndex == NoNode)
-            continue;
+    if (!m_exceptionChecks.empty())
+        m_exceptionChecks.link(this);

 
 

-        VirtualRegister virtualRegister = graph()[nodeIndex].virtualRegister();
+    // lookupExceptionHandler is passed two arguments, the VM and the exec (the CallFrame*).
+    move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);

 
 

-        moveDoubleToPtr(FPRInfo::toRegister(index), GPRInfo::regT0);
-        subPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
-        storePtr(GPRInfo::regT0, addressFor(virtualRegister));
-    }
+    if (doLookup.isSet())
+        doLookup.link(this);

 
 

-    // Fill all FPRs in use by the non-speculative path.
-    for (unsigned index = 0; index < FPRInfo::numberOfRegisters; ++index) {
-        NodeIndex nodeIndex = entry.m_fprInfo[index];
-        if (nodeIndex == NoNode)
-            continue;
+    move(TrustedImmPtr(vm()), GPRInfo::argumentGPR0);
+
+#if CPU(X86)
+    // FIXME: should use the call abstraction, but this is currently in the SpeculativeJIT layer!
+    poke(GPRInfo::argumentGPR0);
+    poke(GPRInfo::argumentGPR1, 1);
+#endif
+    m_calls.append(CallLinkRecord(call(), lookupExceptionHandler));
+    jumpToExceptionHandler();
+}
+
+void JITCompiler::link(LinkBuffer& linkBuffer)
+{
+    // Link the code, populate data in CodeBlock data structures.
+    m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount();
+    m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit();
+
+    if (!m_graph.m_plan.inlineCallFrames->isEmpty())
+        m_jitCode->common.inlineCallFrames = m_graph.m_plan.inlineCallFrames;
+    
+    m_jitCode->common.machineCaptureStart = m_graph.m_machineCaptureStart;
+    m_jitCode->common.slowArguments = WTF::move(m_graph.m_slowArguments);

 
 

-        fillNumericToDouble(nodeIndex, FPRInfo::toRegister(index), GPRInfo::regT0);
+#if USE(JSVALUE32_64)
+    m_jitCode->common.doubleConstants = WTF::move(m_graph.m_doubleConstants);
+#endif
+
+    BitVector usedJumpTables;
+    for (Bag<SwitchData>::iterator iter = m_graph.m_switchData.begin(); !!iter; ++iter) {
+        SwitchData& data = **iter;
+        if (!data.didUseJumpTable)
+            continue;
+        
+        if (data.kind == SwitchString)
+            continue;
+        
+        RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar);
+        
+        usedJumpTables.set(data.switchTableIndex);
+        SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex);
+        table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]);
+        table.ctiOffsets.grow(table.branchOffsets.size());
+        for (unsigned j = table.ctiOffsets.size(); j--;)
+            table.ctiOffsets[j] = table.ctiDefault;
+        for (unsigned j = data.cases.size(); j--;) {
+            SwitchCase& myCase = data.cases[j];
+            table.ctiOffsets[myCase.value.switchLookupValue() - table.min] =
+                linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]);
+        }
+    }
+    
+    for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) {
+        if (usedJumpTables.get(i))
+            continue;
+        
+        m_codeBlock->switchJumpTable(i).clear();

     }
 
     }
 

-    // Fill all GPRs in use by the non-speculative path.
-    for (unsigned index = 0; index < GPRInfo::numberOfRegisters; ++index) {
-        NodeIndex nodeIndex = entry.m_gprInfo[index].nodeIndex;
-        if (nodeIndex == NoNode)
+    // NOTE: we cannot clear string switch tables because (1) we're running concurrently
+    // and we cannot deref StringImpl's and (2) it would be weird to deref those
+    // StringImpl's since we refer to them.
+    for (Bag<SwitchData>::iterator switchDataIter = m_graph.m_switchData.begin(); !!switchDataIter; ++switchDataIter) {
+        SwitchData& data = **switchDataIter;
+        if (!data.didUseJumpTable)

             continue;
             continue;

+        
+        if (data.kind != SwitchString)
+            continue;
+        
+        StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex);
+        table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]);
+        StringJumpTable::StringOffsetTable::iterator iter;
+        StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end();
+        for (iter = table.offsetTable.begin(); iter != end; ++iter)
+            iter->value.ctiOffset = table.ctiDefault;
+        for (unsigned j = data.cases.size(); j--;) {
+            SwitchCase& myCase = data.cases[j];
+            iter = table.offsetTable.find(myCase.value.stringImpl());
+            RELEASE_ASSERT(iter != end);
+            iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]);
+        }
+    }
+
+    // Link all calls out from the JIT code to their respective functions.
+    for (unsigned i = 0; i < m_calls.size(); ++i)
+        linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);

 
 

-        DataFormat dataFormat = entry.m_gprInfo[index].format;
-        if (dataFormat == DataFormatInteger)
-            fillInt32ToInteger(nodeIndex, GPRInfo::toRegister(index));
-        else {
-            ASSERT(dataFormat & DataFormatJS || dataFormat == DataFormatCell); // Treat cell as JSValue for now!
-            fillToJS(nodeIndex, GPRInfo::toRegister(index));
-            // FIXME: For subtypes of DataFormatJS, should jitAssert the subtype?
+    for (unsigned i = m_getByIds.size(); i--;)
+        m_getByIds[i].finalize(linkBuffer);
+    for (unsigned i = m_putByIds.size(); i--;)
+        m_putByIds[i].finalize(linkBuffer);
+
+    for (unsigned i = 0; i < m_ins.size(); ++i) {
+        StructureStubInfo& info = *m_ins[i].m_stubInfo;
+        CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call());
+        info.patch.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_done));
+        info.patch.deltaCallToJump = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_jump));
+        info.callReturnLocation = callReturnLocation;
+        info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label()));
+    }
+    
+    for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
+        JSCallRecord& record = m_jsCalls[i];
+        CallLinkInfo& info = *record.m_info;
+        ThunkGenerator generator = linkThunkGeneratorFor(
+            info.callType == CallLinkInfo::Construct ? CodeForConstruct : CodeForCall,
+            RegisterPreservationNotRequired);
+        linkBuffer.link(record.m_slowCall, FunctionPtr(m_vm->getCTIStub(generator).code().executableAddress()));
+        info.callReturnLocation = linkBuffer.locationOfNearCall(record.m_slowCall);
+        info.hotPathBegin = linkBuffer.locationOf(record.m_targetToCheck);
+        info.hotPathOther = linkBuffer.locationOfNearCall(record.m_fastCall);
+    }
+    
+    MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator);
+    CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
+    for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) {
+        OSRExit& exit = m_jitCode->osrExit[i];
+        OSRExitCompilationInfo& info = m_exitCompilationInfo[i];
+        linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target);
+        exit.correctJump(linkBuffer);
+        if (info.m_replacementSource.isSet()) {
+            m_jitCode->common.jumpReplacements.append(JumpReplacement(
+                linkBuffer.locationOf(info.m_replacementSource),
+                linkBuffer.locationOf(info.m_replacementDestination)));

         }
     }
         }
     }

+    
+    if (m_graph.compilation()) {
+        ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size());
+        for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) {
+            Vector<Label>& labels = m_exitSiteLabels[i];
+            Vector<const void*> addresses;
+            for (unsigned j = 0; j < labels.size(); ++j)
+                addresses.append(linkBuffer.locationOf(labels[j]).executableAddress());
+            m_graph.compilation()->addOSRExitSite(addresses);
+        }
+    } else
+        ASSERT(!m_exitSiteLabels.size());
+    
+    m_jitCode->common.compilation = m_graph.compilation();
+    
+}

 
 

-    // Jump into the non-speculative path.
-    jump(entry.m_entry);
+void JITCompiler::compile()
+{
+    SamplingRegion samplingRegion("DFG Backend");
+
+    setStartOfCode();
+    compileEntry();
+    m_speculative = adoptPtr(new SpeculativeJIT(*this));
+    addPtr(TrustedImm32(m_graph.stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, stackPointerRegister);
+    checkStackPointerAlignment();
+    compileBody();
+    setEndOfMainPath();
+
+    // Generate slow path code.
+    m_speculative->runSlowPathGenerators();
+    
+    compileExceptionHandlers();
+    linkOSRExits();
+    
+    // Create OSR entry trampolines if necessary.
+    m_speculative->createOSREntries();
+    setEndOfCode();

 }
 
 }
 

-void JITCompiler::linkSpeculationChecks(SpeculativeJIT& speculative, NonSpeculativeJIT& nonSpeculative)
+void JITCompiler::link()

 {
 {

-    // Iterators to walk over the set of bail outs & corresponding entry points.
-    SpeculationCheckVector::Iterator checksIter = speculative.speculationChecks().begin();
-    SpeculationCheckVector::Iterator checksEnd = speculative.speculationChecks().end();
-    NonSpeculativeJIT::EntryLocationVector::Iterator entriesIter = nonSpeculative.entryLocations().begin();
-    NonSpeculativeJIT::EntryLocationVector::Iterator entriesEnd = nonSpeculative.entryLocations().end();
-
-    // Iterate over the speculation checks.
-    while (checksIter != checksEnd) {
-        // For every bail out from the speculative path, we must have provided an entry point
-        // into the non-speculative one.
-        ASSERT(checksIter->m_nodeIndex == entriesIter->m_nodeIndex);
-
-        // There may be multiple bail outs that map to the same entry point!
-        do {
-            ASSERT(checksIter != checksEnd);
-            ASSERT(entriesIter != entriesEnd);
-
-            // Plant code to link this speculation failure.
-            const SpeculationCheck& check = *checksIter;
-            const EntryLocation& entry = *entriesIter;
-            jumpFromSpeculativeToNonSpeculative(check, entry, speculative.speculationRecovery(check.m_recoveryIndex));
-             ++checksIter;
-        } while (checksIter != checksEnd && checksIter->m_nodeIndex == entriesIter->m_nodeIndex);
-         ++entriesIter;
+    OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(*m_vm, *this, m_codeBlock, JITCompilationCanFail));
+    if (linkBuffer->didFailToAllocate()) {
+        m_graph.m_plan.finalizer = adoptPtr(new FailedFinalizer(m_graph.m_plan));
+        return;

     }
     }

-
-    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=56289
-    ASSERT(!(checksIter != checksEnd));
-    ASSERT(!(entriesIter != entriesEnd));
+    
+    link(*linkBuffer);
+    m_speculative->linkOSREntries(*linkBuffer);
+    
+    m_jitCode->shrinkToFit();
+    codeBlock()->shrinkToFit(CodeBlock::LateShrink);
+
+    disassemble(*linkBuffer);
+    
+    m_graph.m_plan.finalizer = adoptPtr(new JITFinalizer(
+        m_graph.m_plan, m_jitCode.release(), linkBuffer.release()));

 }
 
 }
 

-void JITCompiler::compileFunction(JITCode& entry, MacroAssemblerCodePtr& entryWithArityCheck)
+void JITCompiler::compileFunction()

 {
 {

-    // === Stage 1 - Function header code generation ===
-    //
-    // This code currently matches the old JIT. In the function header we need to
-    // pop the return address (since we do not allow any recursion on the machine
-    // stack), and perform a fast register file check.
+    SamplingRegion samplingRegion("DFG Backend");
+    
+    setStartOfCode();
+    compileEntry();

 
 

+    // === Function header code generation ===

     // This is the main entry point, without performing an arity check.
     // This is the main entry point, without performing an arity check.

-    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=56292
-    // We'll need to convert the remaining cti_ style calls (specifically the register file
-    // check) which will be dependent on stack layout. (We'd need to account for this in
-    // both normal return code and when jumping to an exception handler).
-    preserveReturnAddressAfterCall(GPRInfo::regT2);
-    emitPutToCallFrameHeader(GPRInfo::regT2, RegisterFile::ReturnPC);

     // If we needed to perform an arity check we will already have moved the return address,
     // so enter after this.
     Label fromArityCheck(this);
     // If we needed to perform an arity check we will already have moved the return address,
     // so enter after this.
     Label fromArityCheck(this);

+    // Plant a check that sufficient space is available in the JSStack.
+    addPtr(TrustedImm32(virtualRegisterForLocal(m_graph.requiredRegisterCountForExecutionAndExit() - 1).offset() * sizeof(Register)), GPRInfo::callFrameRegister, GPRInfo::regT1);
+    Jump stackOverflow = branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), GPRInfo::regT1);

 
 

-    // Setup a pointer to the codeblock in the CallFrameHeader.
-    emitPutImmediateToCallFrameHeader(m_codeBlock, RegisterFile::CodeBlock);
+    // Move the stack pointer down to accommodate locals
+    addPtr(TrustedImm32(m_graph.stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, stackPointerRegister);
+    checkStackPointerAlignment();

 
 

-    // Plant a check that sufficient space is available in the RegisterFile.
-    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=56291
-    addPtr(Imm32(m_codeBlock->m_numCalleeRegisters * sizeof(Register)), GPRInfo::callFrameRegister, GPRInfo::regT1);
-    Jump registerFileCheck = branchPtr(Below, AbsoluteAddress(m_globalData->interpreter->registerFile().addressOfEnd()), GPRInfo::regT1);
-    // Return here after register file check.
-    Label fromRegisterFileCheck = label();
+    // === Function body code generation ===
+    m_speculative = adoptPtr(new SpeculativeJIT(*this));
+    compileBody();
+    setEndOfMainPath();

 
 

-
-    // === Stage 2 - Function body code generation ===
+    // === Function footer code generation ===

     //
     //

-    // We generate the speculative code path, followed by the non-speculative
-    // code for the function. Next we need to link the two together, making
-    // bail-outs from the speculative path jump to the corresponding point on
-    // the non-speculative one (and generating any code necessary to juggle
-    // register values around, rebox values, and ensure spilled, to match the
-    // non-speculative path's requirements).
-
-#if DFG_JIT_BREAK_ON_EVERY_FUNCTION
-    // Handy debug tool!
-    breakpoint();
-#endif
-
-    // First generate the speculative path.
-    Label speculativePathBegin = label();
-    SpeculativeJIT speculative(*this);
-#if !DFG_DEBUG_LOCAL_DISBALE_SPECULATIVE
-    bool compiledSpeculative = speculative.compile();
-#else
-    bool compiledSpeculative = false;
-#endif
-
-    // Next, generate the non-speculative path. We pass this a SpeculationCheckIndexIterator
-    // to allow it to check which nodes in the graph may bail out, and may need to reenter the
-    // non-speculative path.
-    if (compiledSpeculative) {
-        SpeculationCheckIndexIterator checkIterator(speculative.speculationChecks());
-        NonSpeculativeJIT nonSpeculative(*this);
-        nonSpeculative.compile(checkIterator);
-
-        // Link the bail-outs from the speculative path to the corresponding entry points into the non-speculative one.
-        linkSpeculationChecks(speculative, nonSpeculative);
-    } else {
-        // If compilation through the SpeculativeJIT failed, throw away the code we generated.
-        m_calls.clear();
-        rewindToLabel(speculativePathBegin);
-
-        SpeculationCheckVector noChecks;
-        SpeculationCheckIndexIterator checkIterator(noChecks);
-        NonSpeculativeJIT nonSpeculative(*this);
-        nonSpeculative.compile(checkIterator);
-    }
-
-    // === Stage 3 - Function footer code generation ===
+    // Generate code to perform the stack overflow handling (if the stack check in
+    // the function header fails), and generate the entry point with arity check.

     //
     //

-    // Generate code to lookup and jump to exception handlers, to perform the slow
-    // register file check (if the fast one in the function header fails), and
-    // generate the entry point with arity check.
-
-    // Iterate over the m_calls vector, checking for exception checks,
-    // and linking them to here.
-    unsigned exceptionCheckCount = 0;
-    for (unsigned i = 0; i < m_calls.size(); ++i) {
-        Jump& exceptionCheck = m_calls[i].m_exceptionCheck;
-        if (exceptionCheck.isSet()) {
-            exceptionCheck.link(this);
-            ++exceptionCheckCount;
-        }
-    }
-    // If any exception checks were linked, generate code to lookup a handler.
-    if (exceptionCheckCount) {
-        // lookupExceptionHandler is passed two arguments, exec (the CallFrame*), and
-        // an identifier for the operation that threw the exception, which we can use
-        // to look up handler information. The identifier we use is the return address
-        // of the call out from JIT code that threw the exception; this is still
-        // available on the stack, just below the stack pointer!
-        move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
-        peek(GPRInfo::argumentGPR1, -1);
-        m_calls.append(CallRecord(call(), lookupExceptionHandler));
-        // lookupExceptionHandler leaves the handler CallFrame* in the returnValueGPR,
-        // and the address of the handler in returnValueGPR2.
-        jump(GPRInfo::returnValueGPR2);
-    }
+    // Generate the stack overflow handling; if the stack check in the function head fails,
+    // we need to call out to a helper function to throw the StackOverflowError.
+    stackOverflow.link(this);

 
 

-    // Generate the register file check; if the fast check in the function head fails,
-    // we need to call out to a helper function to check whether more space is available.
-    // FIXME: change this from a cti call to a DFG style operation (normal C calling conventions).
-    registerFileCheck.link(this);
-    move(stackPointerRegister, GPRInfo::argumentGPR0);
-    poke(GPRInfo::callFrameRegister, OBJECT_OFFSETOF(struct JITStackFrame, callFrame) / sizeof(void*));
-    Call callRegisterFileCheck = call();
-    jump(fromRegisterFileCheck);
+    emitStoreCodeOrigin(CodeOrigin(0));
+
+    if (maxFrameExtentForSlowPathCall)
+        addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);

 
 

+    m_speculative->callOperationWithCallFrameRollbackOnException(operationThrowStackOverflowError, m_codeBlock);
+    

     // The fast entry point into a function does not check the correct number of arguments
     // have been passed to the call (we only use the fast entry point where we can statically
     // determine the correct number of arguments have been passed, or have already checked).
     // In cases where an arity check is necessary, we enter here.
     // FIXME: change this from a cti call to a DFG style operation (normal C calling conventions).
     // The fast entry point into a function does not check the correct number of arguments
     // have been passed to the call (we only use the fast entry point where we can statically
     // determine the correct number of arguments have been passed, or have already checked).
     // In cases where an arity check is necessary, we enter here.
     // FIXME: change this from a cti call to a DFG style operation (normal C calling conventions).

-    Label arityCheck = label();
-    preserveReturnAddressAfterCall(GPRInfo::regT2);
-    emitPutToCallFrameHeader(GPRInfo::regT2, RegisterFile::ReturnPC);
-    branch32(Equal, GPRInfo::regT1, Imm32(m_codeBlock->m_numParameters)).linkTo(fromArityCheck, this);
-    move(stackPointerRegister, GPRInfo::argumentGPR0);
-    poke(GPRInfo::callFrameRegister, OBJECT_OFFSETOF(struct JITStackFrame, callFrame) / sizeof(void*));
-    Call callArityCheck = call();
-    move(GPRInfo::regT0, GPRInfo::callFrameRegister);
-    jump(fromArityCheck);
-
-
-    // === Stage 4 - Link ===
-    //
-    // Link the code, populate data in CodeBlock data structures.
-
-    LinkBuffer linkBuffer(*m_globalData, this, m_globalData->executableAllocator);
-
-#if DFG_DEBUG_VERBOSE
-    fprintf(stderr, "JIT code start at %p\n", linkBuffer.debugAddress());
-#endif
-
-    // Link all calls out from the JIT code to their respective functions.
-    for (unsigned i = 0; i < m_calls.size(); ++i)
-        linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
-
-    if (m_codeBlock->needsCallReturnIndices()) {
-        m_codeBlock->callReturnIndexVector().reserveCapacity(exceptionCheckCount);
-        for (unsigned i = 0; i < m_calls.size(); ++i) {
-            if (m_calls[i].m_exceptionCheck.isSet()) {
-                unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_calls[i].m_call);
-                unsigned exceptionInfo = m_calls[i].m_exceptionInfo;
-                m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo));
-            }
-        }
-    }
-
-    // FIXME: switch the register file check & arity check over to DFGOpertaion style calls, not JIT stubs.
-    linkBuffer.link(callRegisterFileCheck, cti_register_file_check);
-    linkBuffer.link(callArityCheck, m_codeBlock->m_isConstructor ? cti_op_construct_arityCheck : cti_op_call_arityCheck);
-
-    entryWithArityCheck = linkBuffer.locationOf(arityCheck);
-    entry = linkBuffer.finalizeCode();
-}
-
-#if DFG_JIT_ASSERT
-void JITCompiler::jitAssertIsInt32(GPRReg gpr)
-{
-#if CPU(X86_64)
-    Jump checkInt32 = branchPtr(BelowOrEqual, gpr, TrustedImmPtr(reinterpret_cast<void*>(static_cast<uintptr_t>(0xFFFFFFFFu))));
-    breakpoint();
-    checkInt32.link(this);
+    m_arityCheck = label();
+    compileEntry();
+
+    load32(AssemblyHelpers::payloadFor((VirtualRegister)JSStack::ArgumentCount), GPRInfo::regT1);
+    branch32(AboveOrEqual, GPRInfo::regT1, TrustedImm32(m_codeBlock->numParameters())).linkTo(fromArityCheck, this);
+    emitStoreCodeOrigin(CodeOrigin(0));
+    if (maxFrameExtentForSlowPathCall)
+        addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);
+    m_speculative->callOperationWithCallFrameRollbackOnException(m_codeBlock->m_isConstructor ? operationConstructArityCheck : operationCallArityCheck, GPRInfo::regT0);
+    if (maxFrameExtentForSlowPathCall)
+        addPtr(TrustedImm32(maxFrameExtentForSlowPathCall), stackPointerRegister);
+    branchTest32(Zero, GPRInfo::regT0).linkTo(fromArityCheck, this);
+    emitStoreCodeOrigin(CodeOrigin(0));
+    GPRReg thunkReg;
+#if USE(JSVALUE64)
+    thunkReg = GPRInfo::regT7;

 #else
 #else

-    UNUSED_PARAM(gpr);
-#endif
-}
-
-void JITCompiler::jitAssertIsJSInt32(GPRReg gpr)
-{
-    Jump checkJSInt32 = branchPtr(AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
-    breakpoint();
-    checkJSInt32.link(this);
-}
-
-void JITCompiler::jitAssertIsJSNumber(GPRReg gpr)
-{
-    Jump checkJSNumber = branchTestPtr(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
-    breakpoint();
-    checkJSNumber.link(this);
-}
-
-void JITCompiler::jitAssertIsJSDouble(GPRReg gpr)
-{
-    Jump checkJSInt32 = branchPtr(AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister);
-    Jump checkJSNumber = branchTestPtr(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister);
-    checkJSInt32.link(this);
-    breakpoint();
-    checkJSNumber.link(this);
-}
+    thunkReg = GPRInfo::regT5;

 #endif
 #endif

-
-#if ENABLE(SAMPLING_COUNTERS) && CPU(X86_64) // Or any other 64-bit platform!
-void JITCompiler::emitCount(AbstractSamplingCounter& counter, uint32_t increment)
-{
-    addPtr(TrustedImm32(increment), AbsoluteAddress(counter.addressOfCounter()));
+    move(TrustedImmPtr(m_vm->arityCheckFailReturnThunks->returnPCsFor(*m_vm, m_codeBlock->numParameters())), thunkReg);
+    loadPtr(BaseIndex(thunkReg, GPRInfo::regT0, timesPtr()), thunkReg);
+    m_callArityFixup = call();
+    jump(fromArityCheck);
+    
+    // Generate slow path code.
+    m_speculative->runSlowPathGenerators();
+    
+    compileExceptionHandlers();
+    linkOSRExits();
+    
+    // Create OSR entry trampolines if necessary.
+    m_speculative->createOSREntries();
+    setEndOfCode();

 }
 }

-#endif

 
 

-#if ENABLE(SAMPLING_COUNTERS) && CPU(X86) // Or any other little-endian 32-bit platform!
-void JITCompiler::emitCount(AbstractSamplingCounter& counter, uint32_t increment)
+void JITCompiler::linkFunction()

 {
 {

-    intptr_t hiWord = reinterpret_cast<intptr_t>(counter.addressOfCounter()) + sizeof(int32_t);
-    add32(TrustedImm32(increment), AbsoluteAddress(counter.addressOfCounter()));
-    addWithCarry32(TrustedImm32(0), AbsoluteAddress(reinterpret_cast<void*>(hiWord)));
+    // === Link ===
+    OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(*m_vm, *this, m_codeBlock, JITCompilationCanFail));
+    if (linkBuffer->didFailToAllocate()) {
+        m_graph.m_plan.finalizer = adoptPtr(new FailedFinalizer(m_graph.m_plan));
+        return;
+    }
+    link(*linkBuffer);
+    m_speculative->linkOSREntries(*linkBuffer);
+    
+    m_jitCode->shrinkToFit();
+    codeBlock()->shrinkToFit(CodeBlock::LateShrink);
+    
+    linkBuffer->link(m_callArityFixup, FunctionPtr((m_vm->getCTIStub(arityFixup)).code().executableAddress()));
+    
+    disassemble(*linkBuffer);
+
+    MacroAssemblerCodePtr withArityCheck = linkBuffer->locationOf(m_arityCheck);
+
+    m_graph.m_plan.finalizer = adoptPtr(new JITFinalizer(
+        m_graph.m_plan, m_jitCode.release(), linkBuffer.release(), withArityCheck));

 }
 }

-#endif

 
 

-#if ENABLE(SAMPLING_FLAGS)
-void JITCompiler::setSamplingFlag(int32_t flag)
+void JITCompiler::disassemble(LinkBuffer& linkBuffer)

 {
 {

-    ASSERT(flag >= 1);
-    ASSERT(flag <= 32);
-    or32(TrustedImm32(1u << (flag - 1)), AbsoluteAddress(SamplingFlags::addressOfFlags()));
+    if (shouldShowDisassembly())
+        m_disassembler->dump(linkBuffer);
+    
+    if (m_graph.m_plan.compilation)
+        m_disassembler->reportToProfiler(m_graph.m_plan.compilation.get(), linkBuffer);

 }
 
 }
 

-void JITCompiler::clearSamplingFlag(int32_t flag)
+#if USE(JSVALUE32_64)
+void* JITCompiler::addressOfDoubleConstant(Node* node)

 {
 {

-    ASSERT(flag >= 1);
-    ASSERT(flag <= 32);
-    and32(TrustedImm32(~(1u << (flag - 1))), AbsoluteAddress(SamplingFlags::addressOfFlags()));
+    ASSERT(m_graph.isNumberConstant(node));
+    JSValue jsvalue = node->valueOfJSConstant(codeBlock());
+    ASSERT(jsvalue.isDouble());
+
+    double value = jsvalue.asDouble();
+    int64_t valueBits = bitwise_cast<int64_t>(value);
+    auto it = m_graph.m_doubleConstantsMap.find(valueBits);
+    if (it != m_graph.m_doubleConstantsMap.end())
+        return it->second;
+
+    if (!m_graph.m_doubleConstants)
+        m_graph.m_doubleConstants = std::make_unique<Bag<double>>();
+
+    double* addressInConstantPool = m_graph.m_doubleConstants->add();
+    *addressInConstantPool = value;
+    m_graph.m_doubleConstantsMap[valueBits] = addressInConstantPool;
+    return addressInConstantPool;

 }
 #endif
 
 } } // namespace JSC::DFG
 
 }
 #endif
 
 } } // namespace JSC::DFG
 

-#endif
+#endif // ENABLE(DFG_JIT)