[apple/javascriptcore.git] / ftl / FTLOSRExitCompiler.cpp

/*
 * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

#include "config.h"
#include "FTLOSRExitCompiler.h"

#if ENABLE(FTL_JIT)

#include "DFGOSRExitCompilerCommon.h"
#include "DFGOSRExitPreparation.h"
#include "FTLExitArgumentForOperand.h"
#include "FTLJITCode.h"
#include "FTLOSRExit.h"
#include "FTLState.h"
#include "FTLSaveRestore.h"
#include "LinkBuffer.h"
#include "MaxFrameExtentForSlowPathCall.h"
#include "OperandsInlines.h"
#include "JSCInlines.h"
#include "RegisterPreservationWrapperGenerator.h"
#include "RepatchBuffer.h"

namespace JSC { namespace FTL {

using namespace DFG;

static void compileStub(
    unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
    StackMaps::Record* record = nullptr;
    
    for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
        record = &jitCode->stackmaps.records[i];
        if (record->patchpointID == exit.m_stackmapID)
            break;
    }
    
    RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
    
    // This code requires framePointerRegister is the same as callFrameRegister
    static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");

    CCallHelpers jit(vm, codeBlock);
    
    // We need scratch space to save all registers and to build up the JSStack.
    // Use a scratch buffer to transfer all values.
    ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size() + requiredScratchMemorySizeInBytes() + jitCode->unwindInfo.m_registers.size() * sizeof(uint64_t));
    EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
    char* registerScratch = bitwise_cast<char*>(scratch + exit.m_values.size());
    uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
    
    // Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
    // We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
    // that slot for saveAllRegisters().

    saveAllRegisters(jit, registerScratch);
    
    // Bring the stack back into a sane form and assert that it's sane.
    jit.popToRestore(GPRInfo::regT0);
    jit.checkStackPointerAlignment();
    
    if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
        Profiler::Database& database = *vm->m_perBytecodeProfiler;
        Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
        
        Profiler::OSRExit* profilerExit = compilation->addOSRExit(
            exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
            exit.m_kind, isWatchpoint(exit.m_kind));
        jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
    }

    // The remaining code assumes that SP/FP are in the same state that they were in the FTL's
    // call frame.
    
    // Get the call frame and tag thingies.
    // Restore the exiting function's callFrame value into a regT4
    jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
    jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
    
    // Do some value profiling.
    if (exit.m_profileValueFormat != InvalidValueFormat) {
        record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
        reboxAccordingToFormat(
            exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
        
        if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
            CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
            if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
                jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
                jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
                jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
                jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
                jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
                jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
            }
        }
        
        if (!!exit.m_valueProfile)
            jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
    }

    // Save all state from wherever the exit data tells us it was, into the appropriate place in
    // the scratch buffer. This doesn't rebox any values yet.
    
    for (unsigned index = exit.m_values.size(); index--;) {
        ExitValue value = exit.m_values[index];
        
        switch (value.kind()) {
        case ExitValueDead:
            jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
            break;
            
        case ExitValueConstant:
            jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
            break;
            
        case ExitValueArgument:
            record->locations[value.exitArgument().argument()].restoreInto(
                jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
            break;
            
        case ExitValueInJSStack:
        case ExitValueInJSStackAsInt32:
        case ExitValueInJSStackAsInt52:
        case ExitValueInJSStackAsDouble:
            jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
            break;
            
        case ExitValueArgumentsObjectThatWasNotCreated:
            // We can't actually recover this yet, but we can make the stack look sane. This is
            // a prerequisite to running the actual arguments recovery.
            jit.move(MacroAssembler::TrustedImm64(JSValue::encode(JSValue())), GPRInfo::regT0);
            break;
            
        case ExitValueRecovery:
            record->locations[value.rightRecoveryArgument()].restoreInto(
                jit, jitCode->stackmaps, registerScratch, GPRInfo::regT1);
            record->locations[value.leftRecoveryArgument()].restoreInto(
                jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
            switch (value.recoveryOpcode()) {
            case AddRecovery:
                switch (value.recoveryFormat()) {
                case ValueFormatInt32:
                    jit.add32(GPRInfo::regT1, GPRInfo::regT0);
                    break;
                case ValueFormatInt52:
                    jit.add64(GPRInfo::regT1, GPRInfo::regT0);
                    break;
                default:
                    RELEASE_ASSERT_NOT_REACHED();
                    break;
                }
                break;
            case SubRecovery:
                switch (value.recoveryFormat()) {
                case ValueFormatInt32:
                    jit.sub32(GPRInfo::regT1, GPRInfo::regT0);
                    break;
                case ValueFormatInt52:
                    jit.sub64(GPRInfo::regT1, GPRInfo::regT0);
                    break;
                default:
                    RELEASE_ASSERT_NOT_REACHED();
                    break;
                }
                break;
            default:
                RELEASE_ASSERT_NOT_REACHED();
                break;
            }
            break;
            
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
        
        jit.store64(GPRInfo::regT0, scratch + index);
    }
    
    // Henceforth we make it look like the exiting function was called through a register
    // preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
    // we restore the various things according to either exit.m_values or by copying from the
    // old frame, and finally we save the various callee-save registers into where the
    // restoration thunk would restore them from.
    
    ptrdiff_t offset = registerPreservationOffset();
    RegisterSet toSave = registersToPreserve();
    
    // Before we start messing with the frame, we need to set aside any registers that the
    // FTL code was preserving.
    for (unsigned i = jitCode->unwindInfo.m_registers.size(); i--;) {
        RegisterAtOffset entry = jitCode->unwindInfo.m_registers[i];
        jit.load64(
            MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
            GPRInfo::regT0);
        jit.store64(GPRInfo::regT0, unwindScratch + i);
    }
    
    jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
    
    // Let's say that the FTL function had failed its arity check. In that case, the stack will
    // contain some extra stuff.
    //
    // First we compute the padded stack space:
    //
    //     paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
    //
    // The stack will have regT2 + CallFrameHeaderSize stuff, but above it there will be
    // paddedStackSpace gunk used by the arity check fail restoration thunk. When that happens
    // we want to make the stack look like this, from higher addresses down:
    //
    //     - register preservation return PC
    //     - preserved registers
    //     - arity check fail return PC
    //     - argument padding
    //     - actual arguments
    //     - call frame header
    //
    // So that the actual call frame header appears to return to the arity check fail return
    // PC, and that then returns to the register preservation thunk. The arity check thunk that
    // we return to will have the padding size encoded into it. It will then know to return
    // into the register preservation thunk, which uses the argument count to figure out where
    // registers are preserved.

    // This code assumes that we're dealing with FunctionCode.
    RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
    
    jit.add32(
        MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
        GPRInfo::regT3);
    MacroAssembler::Jump arityIntact = jit.branch32(
        MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
    jit.neg32(GPRInfo::regT3);
    jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
    jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
    jit.add32(GPRInfo::regT3, GPRInfo::regT2);
    arityIntact.link(&jit);

    // First set up SP so that our data doesn't get clobbered by signals.
    unsigned conservativeStackDelta =
        registerPreservationOffset() +
        exit.m_values.numberOfLocals() * sizeof(Register) +
        maxFrameExtentForSlowPathCall;
    conservativeStackDelta = WTF::roundUpToMultipleOf(
        stackAlignmentBytes(), conservativeStackDelta);
    jit.addPtr(
        MacroAssembler::TrustedImm32(-conservativeStackDelta),
        MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
    jit.checkStackPointerAlignment();
    
    jit.subPtr(
        MacroAssembler::TrustedImm32(registerPreservationOffset()),
        MacroAssembler::framePointerRegister);
    
    // Copy the old frame data into its new location.
    jit.add32(MacroAssembler::TrustedImm32(JSStack::CallFrameHeaderSize), GPRInfo::regT2);
    jit.move(MacroAssembler::framePointerRegister, GPRInfo::regT1);
    MacroAssembler::Label loop = jit.label();
    jit.sub32(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
    jit.load64(MacroAssembler::Address(GPRInfo::regT1, offset), GPRInfo::regT0);
    jit.store64(GPRInfo::regT0, GPRInfo::regT1);
    jit.addPtr(MacroAssembler::TrustedImm32(sizeof(Register)), GPRInfo::regT1);
    jit.branchTest32(MacroAssembler::NonZero, GPRInfo::regT2).linkTo(loop, &jit);
    
    // At this point regT1 points to where we would save our registers. Save them here.
    ptrdiff_t currentOffset = 0;
    for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
        if (!toSave.get(reg))
            continue;
        currentOffset += sizeof(Register);
        unsigned unwindIndex = jitCode->unwindInfo.indexOf(reg);
        if (unwindIndex == UINT_MAX) {
            // The FTL compilation didn't preserve this register. This means that it also
            // didn't use the register. So its value at the beginning of OSR exit should be
            // preserved by the thunk. Luckily, we saved all registers into the register
            // scratch buffer, so we can restore them from there.
            jit.load64(registerScratch + offsetOfReg(reg), GPRInfo::regT0);
        } else {
            // The FTL compilation preserved the register. Its new value is therefore
            // irrelevant, but we can get the value that was preserved by using the unwind
            // data. We've already copied all unwind-able preserved registers into the unwind
            // scratch buffer, so we can get it from there.
            jit.load64(unwindScratch + unwindIndex, GPRInfo::regT0);
        }
        jit.store64(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT1, currentOffset));
    }
    
    // We need to make sure that we return into the register restoration thunk. This works
    // differently depending on whether or not we had arity issues.
    MacroAssembler::Jump arityIntactForReturnPC = jit.branch32(
        MacroAssembler::GreaterThanOrEqual,
        CCallHelpers::payloadFor(JSStack::ArgumentCount),
        MacroAssembler::TrustedImm32(codeBlock->numParameters()));
    
    // The return PC in the call frame header points at exactly the right arity restoration
    // thunk. We don't want to change that. But the arity restoration thunk's frame has a
    // return PC and we want to reroute that to our register restoration thunk. The arity
    // restoration's return PC just just below regT1, and the register restoration's return PC
    // is right at regT1.
    jit.loadPtr(MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))), GPRInfo::regT0);
    jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
    jit.storePtr(
        MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
        MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))));
    
    MacroAssembler::Jump arityReturnPCReady = jit.jump();

    arityIntactForReturnPC.link(&jit);
    
    jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), GPRInfo::regT0);
    jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
    jit.storePtr(
        MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
        MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()));
    
    arityReturnPCReady.link(&jit);
    
    // Now get state out of the scratch buffer and place it back into the stack. This part does
    // all reboxing.
    for (unsigned index = exit.m_values.size(); index--;) {
        int operand = exit.m_values.operandForIndex(index);
        ExitValue value = exit.m_values[index];
        
        jit.load64(scratch + index, GPRInfo::regT0);
        reboxAccordingToFormat(
            value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
        jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(static_cast<VirtualRegister>(operand)));
    }
    
    handleExitCounts(jit, exit);
    reifyInlinedCallFrames(jit, exit);
    
    ArgumentsRecoveryGenerator argumentsRecovery;
    for (unsigned index = exit.m_values.size(); index--;) {
        if (!exit.m_values[index].isArgumentsObjectThatWasNotCreated())
            continue;
        int operand = exit.m_values.operandForIndex(index);
        argumentsRecovery.generateFor(operand, exit.m_codeOrigin, jit);
    }
    
    adjustAndJumpToTarget(jit, exit);
    
    LinkBuffer patchBuffer(*vm, jit, codeBlock);
    exit.m_code = FINALIZE_CODE_IF(
        shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
        patchBuffer,
        ("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
            exitID, toCString(exit.m_codeOrigin).data(),
            exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
            toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
            toCString(*record).data()));
}

extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID)
{
    SamplingRegion samplingRegion("FTL OSR Exit Compilation");

    if (shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit())
        dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
    
    CodeBlock* codeBlock = exec->codeBlock();
    
    ASSERT(codeBlock);
    ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
    
    VM* vm = &exec->vm();
    
    // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
    // really be profitable.
    DeferGCForAWhile deferGC(vm->heap);

    JITCode* jitCode = codeBlock->jitCode()->ftl();
    OSRExit& exit = jitCode->osrExit[exitID];
    
    prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
    
    compileStub(exitID, jitCode, exit, vm, codeBlock);
    
    RepatchBuffer repatchBuffer(codeBlock);
    repatchBuffer.relink(
        exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
    
    return exit.m_code.code().executableAddress();
}

} } // namespace JSC::FTL

#endif // ENABLE(FTL_JIT)
Commit	Line	Data
81345200 A	1	/*
	2	* Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
	3	*
	4	* Redistribution and use in source and binary forms, with or without
	5	* modification, are permitted provided that the following conditions
	6	* are met:
	7	* 1. Redistributions of source code must retain the above copyright
	8	* notice, this list of conditions and the following disclaimer.
	9	* 2. Redistributions in binary form must reproduce the above copyright
	10	* notice, this list of conditions and the following disclaimer in the
	11	* documentation and/or other materials provided with the distribution.
	12	*
	13	* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
	14	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	15	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	16	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
	17	* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
	18	* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
	19	* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
	20	* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
	21	* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	22	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	23	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	24	*/
	25
	26	#include "config.h"
	27	#include "FTLOSRExitCompiler.h"
	28
	29	#if ENABLE(FTL_JIT)
	30
	31	#include "DFGOSRExitCompilerCommon.h"
	32	#include "DFGOSRExitPreparation.h"
	33	#include "FTLExitArgumentForOperand.h"
	34	#include "FTLJITCode.h"
	35	#include "FTLOSRExit.h"
	36	#include "FTLState.h"
	37	#include "FTLSaveRestore.h"
	38	#include "LinkBuffer.h"
	39	#include "MaxFrameExtentForSlowPathCall.h"
	40	#include "OperandsInlines.h"
	41	#include "JSCInlines.h"
	42	#include "RegisterPreservationWrapperGenerator.h"
	43	#include "RepatchBuffer.h"
	44
	45	namespace JSC { namespace FTL {
	46
	47	using namespace DFG;
	48
	49	static void compileStub(
	50	unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
	51	{
	52	StackMaps::Record* record = nullptr;
	53
	54	for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
	55	record = &jitCode->stackmaps.records[i];
	56	if (record->patchpointID == exit.m_stackmapID)
	57	break;
	58	}
	59
	60	RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
	61
	62	// This code requires framePointerRegister is the same as callFrameRegister
	63	static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
	64
65	CCallHelpers jit(vm, codeBlock);
66
67	// We need scratch space to save all registers and to build up the JSStack.
68	// Use a scratch buffer to transfer all values.
69	ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size() + requiredScratchMemorySizeInBytes() + jitCode->unwindInfo.m_registers.size() * sizeof(uint64_t));
70	EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
71	char* registerScratch = bitwise_cast<char*>(scratch + exit.m_values.size());
72	uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
73
74	// Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
75	// We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
76	// that slot for saveAllRegisters().
77
78	saveAllRegisters(jit, registerScratch);
79
80	// Bring the stack back into a sane form and assert that it's sane.
81	jit.popToRestore(GPRInfo::regT0);
82	jit.checkStackPointerAlignment();
83
84	if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
85	Profiler::Database& database = *vm->m_perBytecodeProfiler;
86	Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
87
88	Profiler::OSRExit* profilerExit = compilation->addOSRExit(
89	exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
90	exit.m_kind, isWatchpoint(exit.m_kind));
91	jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
92	}
93
94	// The remaining code assumes that SP/FP are in the same state that they were in the FTL's
95	// call frame.
96
97	// Get the call frame and tag thingies.
98	// Restore the exiting function's callFrame value into a regT4
99	jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
100	jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
101
102	// Do some value profiling.
103	if (exit.m_profileValueFormat != InvalidValueFormat) {
104	record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
105	reboxAccordingToFormat(
106	exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
107
108	if (exit.m_kind == BadCache \|\| exit.m_kind == BadIndexingType) {
109	CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
110	if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
111	jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
112	jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
113	jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
114	jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
115	jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
116	jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
117	}
118	}
119
120	if (!!exit.m_valueProfile)
121	jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
122	}
123
124	// Save all state from wherever the exit data tells us it was, into the appropriate place in
125	// the scratch buffer. This doesn't rebox any values yet.
126
127	for (unsigned index = exit.m_values.size(); index--;) {
128	ExitValue value = exit.m_values[index];
129
130	switch (value.kind()) {
131	case ExitValueDead:
132	jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
133	break;
134
135	case ExitValueConstant:
136	jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
137	break;
138
139	case ExitValueArgument:
140	record->locations[value.exitArgument().argument()].restoreInto(
141	jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
142	break;
143
144	case ExitValueInJSStack:
145	case ExitValueInJSStackAsInt32:
146	case ExitValueInJSStackAsInt52:
147	case ExitValueInJSStackAsDouble:
148	jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
149	break;
150
151	case ExitValueArgumentsObjectThatWasNotCreated:
152	// We can't actually recover this yet, but we can make the stack look sane. This is
153	// a prerequisite to running the actual arguments recovery.
154	jit.move(MacroAssembler::TrustedImm64(JSValue::encode(JSValue())), GPRInfo::regT0);
155	break;
156
157	case ExitValueRecovery:
158	record->locations[value.rightRecoveryArgument()].restoreInto(
159	jit, jitCode->stackmaps, registerScratch, GPRInfo::regT1);
160	record->locations[value.leftRecoveryArgument()].restoreInto(
161	jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
162	switch (value.recoveryOpcode()) {
163	case AddRecovery:
164	switch (value.recoveryFormat()) {
165	case ValueFormatInt32:
166	jit.add32(GPRInfo::regT1, GPRInfo::regT0);
167	break;
168	case ValueFormatInt52:
169	jit.add64(GPRInfo::regT1, GPRInfo::regT0);
170	break;
171	default:
172	RELEASE_ASSERT_NOT_REACHED();
173	break;
174	}
175	break;
176	case SubRecovery:
177	switch (value.recoveryFormat()) {
178	case ValueFormatInt32:
179	jit.sub32(GPRInfo::regT1, GPRInfo::regT0);
180	break;
181	case ValueFormatInt52:
182	jit.sub64(GPRInfo::regT1, GPRInfo::regT0);
183	break;
184	default:
185	RELEASE_ASSERT_NOT_REACHED();
186	break;
187	}
188	break;
189	default:
190	RELEASE_ASSERT_NOT_REACHED();
191	break;
192	}
193	break;
194
195	default:
196	RELEASE_ASSERT_NOT_REACHED();
197	break;
198	}
199
200	jit.store64(GPRInfo::regT0, scratch + index);
201	}
202
203	// Henceforth we make it look like the exiting function was called through a register
204	// preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
205	// we restore the various things according to either exit.m_values or by copying from the
206	// old frame, and finally we save the various callee-save registers into where the
207	// restoration thunk would restore them from.
208
209	ptrdiff_t offset = registerPreservationOffset();
210	RegisterSet toSave = registersToPreserve();
211
212	// Before we start messing with the frame, we need to set aside any registers that the
213	// FTL code was preserving.
214	for (unsigned i = jitCode->unwindInfo.m_registers.size(); i--;) {
215	RegisterAtOffset entry = jitCode->unwindInfo.m_registers[i];
216	jit.load64(
217	MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
218	GPRInfo::regT0);
219	jit.store64(GPRInfo::regT0, unwindScratch + i);
220	}
221
222	jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
223
224	// Let's say that the FTL function had failed its arity check. In that case, the stack will
225	// contain some extra stuff.
226	//
227	// First we compute the padded stack space:
228	//
229	// paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
230	//
231	// The stack will have regT2 + CallFrameHeaderSize stuff, but above it there will be
232	// paddedStackSpace gunk used by the arity check fail restoration thunk. When that happens
233	// we want to make the stack look like this, from higher addresses down:
234	//
235	// - register preservation return PC
236	// - preserved registers
237	// - arity check fail return PC
238	// - argument padding
239	// - actual arguments
240	// - call frame header
241	//
242	// So that the actual call frame header appears to return to the arity check fail return
243	// PC, and that then returns to the register preservation thunk. The arity check thunk that
244	// we return to will have the padding size encoded into it. It will then know to return
245	// into the register preservation thunk, which uses the argument count to figure out where
246	// registers are preserved.
247
248	// This code assumes that we're dealing with FunctionCode.
249	RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
250
251	jit.add32(
252	MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
253	GPRInfo::regT3);
254	MacroAssembler::Jump arityIntact = jit.branch32(
255	MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
256	jit.neg32(GPRInfo::regT3);
257	jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
258	jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
259	jit.add32(GPRInfo::regT3, GPRInfo::regT2);
260	arityIntact.link(&jit);
261
262	// First set up SP so that our data doesn't get clobbered by signals.
263	unsigned conservativeStackDelta =
264	registerPreservationOffset() +
265	exit.m_values.numberOfLocals() * sizeof(Register) +
266	maxFrameExtentForSlowPathCall;
267	conservativeStackDelta = WTF::roundUpToMultipleOf(
268	stackAlignmentBytes(), conservativeStackDelta);
269	jit.addPtr(
270	MacroAssembler::TrustedImm32(-conservativeStackDelta),
271	MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
272	jit.checkStackPointerAlignment();
273
274	jit.subPtr(
275	MacroAssembler::TrustedImm32(registerPreservationOffset()),
276	MacroAssembler::framePointerRegister);
277
278	// Copy the old frame data into its new location.
279	jit.add32(MacroAssembler::TrustedImm32(JSStack::CallFrameHeaderSize), GPRInfo::regT2);
280	jit.move(MacroAssembler::framePointerRegister, GPRInfo::regT1);
281	MacroAssembler::Label loop = jit.label();
282	jit.sub32(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
283	jit.load64(MacroAssembler::Address(GPRInfo::regT1, offset), GPRInfo::regT0);
284	jit.store64(GPRInfo::regT0, GPRInfo::regT1);
285	jit.addPtr(MacroAssembler::TrustedImm32(sizeof(Register)), GPRInfo::regT1);
286	jit.branchTest32(MacroAssembler::NonZero, GPRInfo::regT2).linkTo(loop, &jit);
287
288	// At this point regT1 points to where we would save our registers. Save them here.
289	ptrdiff_t currentOffset = 0;
290	for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
291	if (!toSave.get(reg))
292	continue;
293	currentOffset += sizeof(Register);
294	unsigned unwindIndex = jitCode->unwindInfo.indexOf(reg);
295	if (unwindIndex == UINT_MAX) {
296	// The FTL compilation didn't preserve this register. This means that it also
297	// didn't use the register. So its value at the beginning of OSR exit should be
298	// preserved by the thunk. Luckily, we saved all registers into the register
299	// scratch buffer, so we can restore them from there.
300	jit.load64(registerScratch + offsetOfReg(reg), GPRInfo::regT0);
301	} else {
302	// The FTL compilation preserved the register. Its new value is therefore
303	// irrelevant, but we can get the value that was preserved by using the unwind
304	// data. We've already copied all unwind-able preserved registers into the unwind
305	// scratch buffer, so we can get it from there.
306	jit.load64(unwindScratch + unwindIndex, GPRInfo::regT0);
307	}
308	jit.store64(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT1, currentOffset));
309	}
310
311	// We need to make sure that we return into the register restoration thunk. This works
312	// differently depending on whether or not we had arity issues.
313	MacroAssembler::Jump arityIntactForReturnPC = jit.branch32(
314	MacroAssembler::GreaterThanOrEqual,
315	CCallHelpers::payloadFor(JSStack::ArgumentCount),
316	MacroAssembler::TrustedImm32(codeBlock->numParameters()));
317
318	// The return PC in the call frame header points at exactly the right arity restoration
319	// thunk. We don't want to change that. But the arity restoration thunk's frame has a
320	// return PC and we want to reroute that to our register restoration thunk. The arity
321	// restoration's return PC just just below regT1, and the register restoration's return PC
322	// is right at regT1.
323	jit.loadPtr(MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))), GPRInfo::regT0);
324	jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
325	jit.storePtr(
326	MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
327	MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))));
328
329	MacroAssembler::Jump arityReturnPCReady = jit.jump();
330
331	arityIntactForReturnPC.link(&jit);
332
333	jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), GPRInfo::regT0);
334	jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
335	jit.storePtr(
336	MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
337	MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()));
338
339	arityReturnPCReady.link(&jit);
340
341	// Now get state out of the scratch buffer and place it back into the stack. This part does
342	// all reboxing.
343	for (unsigned index = exit.m_values.size(); index--;) {
344	int operand = exit.m_values.operandForIndex(index);
345	ExitValue value = exit.m_values[index];
346
347	jit.load64(scratch + index, GPRInfo::regT0);
348	reboxAccordingToFormat(
349	value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
350	jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(static_cast<VirtualRegister>(operand)));
351	}
352
353	handleExitCounts(jit, exit);
354	reifyInlinedCallFrames(jit, exit);
355
356	ArgumentsRecoveryGenerator argumentsRecovery;
357	for (unsigned index = exit.m_values.size(); index--;) {
358	if (!exit.m_values[index].isArgumentsObjectThatWasNotCreated())
359	continue;
360	int operand = exit.m_values.operandForIndex(index);
361	argumentsRecovery.generateFor(operand, exit.m_codeOrigin, jit);
362	}
363
364	adjustAndJumpToTarget(jit, exit);
365
366	LinkBuffer patchBuffer(*vm, jit, codeBlock);
367	exit.m_code = FINALIZE_CODE_IF(
368	shouldShowDisassembly() \|\| Options::verboseOSR() \|\| Options::verboseFTLOSRExit(),
369	patchBuffer,
370	("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
371	exitID, toCString(exit.m_codeOrigin).data(),
372	exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
373	toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
374	toCString(*record).data()));
375	}
376
377	extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID)
378	{
379	SamplingRegion samplingRegion("FTL OSR Exit Compilation");
380
381	if (shouldShowDisassembly() \|\| Options::verboseOSR() \|\| Options::verboseFTLOSRExit())
382	dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
383
384	CodeBlock* codeBlock = exec->codeBlock();
385
386	ASSERT(codeBlock);
387	ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
388
389	VM* vm = &exec->vm();
390
391	// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
392	// really be profitable.
393	DeferGCForAWhile deferGC(vm->heap);
394
395	JITCode* jitCode = codeBlock->jitCode()->ftl();
396	OSRExit& exit = jitCode->osrExit[exitID];
397
398	prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
399
400	compileStub(exitID, jitCode, exit, vm, codeBlock);
401
402	RepatchBuffer repatchBuffer(codeBlock);
403	repatchBuffer.relink(
404	exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
405
406	return exit.m_code.code().executableAddress();
407	}
408
409	} } // namespace JSC::FTL
410
411	#endif // ENABLE(FTL_JIT)
412