ftl/FTLOSRExitCompiler.cpp

   1 /*
   2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLOSRExitCompiler.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "DFGOSRExitCompilerCommon.h"
  32 #include "DFGOSRExitPreparation.h"
  33 #include "FTLExitArgumentForOperand.h"
  34 #include "FTLJITCode.h"
  35 #include "FTLOSRExit.h"
  36 #include "FTLState.h"
  37 #include "FTLSaveRestore.h"
  38 #include "LinkBuffer.h"
  39 #include "MaxFrameExtentForSlowPathCall.h"
  40 #include "OperandsInlines.h"
  41 #include "JSCInlines.h"
  42 #include "RegisterPreservationWrapperGenerator.h"
  43 #include "RepatchBuffer.h"
  44
  45 namespace JSC { namespace FTL {
  46
  47 using namespace DFG;
  48
  49 static void compileStub(
  50     unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
  51 {
  52     StackMaps::Record* record = nullptr;
  53
  54     for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
  55         record = &jitCode->stackmaps.records[i];
  56         if (record->patchpointID == exit.m_stackmapID)
  57             break;
  58     }
  59
  60     RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
  61
  62     // This code requires framePointerRegister is the same as callFrameRegister
  63     static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
  64
  65     CCallHelpers jit(vm, codeBlock);
  66
  67     // We need scratch space to save all registers and to build up the JSStack.
  68     // Use a scratch buffer to transfer all values.
  69     ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size() + requiredScratchMemorySizeInBytes() + jitCode->unwindInfo.m_registers.size() * sizeof(uint64_t));
  70     EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
  71     char* registerScratch = bitwise_cast<char*>(scratch + exit.m_values.size());
  72     uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
  73
  74     // Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
  75     // We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
  76     // that slot for saveAllRegisters().
  77
  78     saveAllRegisters(jit, registerScratch);
  79
  80     // Bring the stack back into a sane form and assert that it's sane.
  81     jit.popToRestore(GPRInfo::regT0);
  82     jit.checkStackPointerAlignment();
  83
  84     if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
  85         Profiler::Database& database = *vm->m_perBytecodeProfiler;
  86         Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
  87
  88         Profiler::OSRExit* profilerExit = compilation->addOSRExit(
  89             exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
  90             exit.m_kind, isWatchpoint(exit.m_kind));
  91         jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
  92     }
  93
  94     // The remaining code assumes that SP/FP are in the same state that they were in the FTL's
  95     // call frame.
  96
  97     // Get the call frame and tag thingies.
  98     // Restore the exiting function's callFrame value into a regT4
  99     jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
 100     jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
 101
 102     // Do some value profiling.
 103     if (exit.m_profileValueFormat != InvalidValueFormat) {
 104         record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
 105         reboxAccordingToFormat(
 106             exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
 107
 108         if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
 109             CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
 110             if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
 111                 jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
 112                 jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
 113                 jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
 114                 jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
 115                 jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
 116                 jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
 117             }
 118         }
 119
 120         if (!!exit.m_valueProfile)
 121             jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
 122     }
 123
 124     // Save all state from wherever the exit data tells us it was, into the appropriate place in
 125     // the scratch buffer. This doesn't rebox any values yet.
 126
 127     for (unsigned index = exit.m_values.size(); index--;) {
 128         ExitValue value = exit.m_values[index];
 129
 130         switch (value.kind()) {
 131         case ExitValueDead:
 132             jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
 133             break;
 134
 135         case ExitValueConstant:
 136             jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
 137             break;
 138
 139         case ExitValueArgument:
 140             record->locations[value.exitArgument().argument()].restoreInto(
 141                 jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
 142             break;
 143
 144         case ExitValueInJSStack:
 145         case ExitValueInJSStackAsInt32:
 146         case ExitValueInJSStackAsInt52:
 147         case ExitValueInJSStackAsDouble:
 148             jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
 149             break;
 150
 151         case ExitValueArgumentsObjectThatWasNotCreated:
 152             // We can't actually recover this yet, but we can make the stack look sane. This is
 153             // a prerequisite to running the actual arguments recovery.
 154             jit.move(MacroAssembler::TrustedImm64(JSValue::encode(JSValue())), GPRInfo::regT0);
 155             break;
 156
 157         case ExitValueRecovery:
 158             record->locations[value.rightRecoveryArgument()].restoreInto(
 159                 jit, jitCode->stackmaps, registerScratch, GPRInfo::regT1);
 160             record->locations[value.leftRecoveryArgument()].restoreInto(
 161                 jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
 162             switch (value.recoveryOpcode()) {
 163             case AddRecovery:
 164                 switch (value.recoveryFormat()) {
 165                 case ValueFormatInt32:
 166                     jit.add32(GPRInfo::regT1, GPRInfo::regT0);
 167                     break;
 168                 case ValueFormatInt52:
 169                     jit.add64(GPRInfo::regT1, GPRInfo::regT0);
 170                     break;
 171                 default:
 172                     RELEASE_ASSERT_NOT_REACHED();
 173                     break;
 174                 }
 175                 break;
 176             case SubRecovery:
 177                 switch (value.recoveryFormat()) {
 178                 case ValueFormatInt32:
 179                     jit.sub32(GPRInfo::regT1, GPRInfo::regT0);
 180                     break;
 181                 case ValueFormatInt52:
 182                     jit.sub64(GPRInfo::regT1, GPRInfo::regT0);
 183                     break;
 184                 default:
 185                     RELEASE_ASSERT_NOT_REACHED();
 186                     break;
 187                 }
 188                 break;
 189             default:
 190                 RELEASE_ASSERT_NOT_REACHED();
 191                 break;
 192             }
 193             break;
 194
 195         default:
 196             RELEASE_ASSERT_NOT_REACHED();
 197             break;
 198         }
 199
 200         jit.store64(GPRInfo::regT0, scratch + index);
 201     }
 202
 203     // Henceforth we make it look like the exiting function was called through a register
 204     // preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
 205     // we restore the various things according to either exit.m_values or by copying from the
 206     // old frame, and finally we save the various callee-save registers into where the
 207     // restoration thunk would restore them from.
 208
 209     ptrdiff_t offset = registerPreservationOffset();
 210     RegisterSet toSave = registersToPreserve();
 211
 212     // Before we start messing with the frame, we need to set aside any registers that the
 213     // FTL code was preserving.
 214     for (unsigned i = jitCode->unwindInfo.m_registers.size(); i--;) {
 215         RegisterAtOffset entry = jitCode->unwindInfo.m_registers[i];
 216         jit.load64(
 217             MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
 218             GPRInfo::regT0);
 219         jit.store64(GPRInfo::regT0, unwindScratch + i);
 220     }
 221
 222     jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
 223
 224     // Let's say that the FTL function had failed its arity check. In that case, the stack will
 225     // contain some extra stuff.
 226     //
 227     // First we compute the padded stack space:
 228     //
 229     //     paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
 230     //
 231     // The stack will have regT2 + CallFrameHeaderSize stuff, but above it there will be
 232     // paddedStackSpace gunk used by the arity check fail restoration thunk. When that happens
 233     // we want to make the stack look like this, from higher addresses down:
 234     //
 235     //     - register preservation return PC
 236     //     - preserved registers
 237     //     - arity check fail return PC
 238     //     - argument padding
 239     //     - actual arguments
 240     //     - call frame header
 241     //
 242     // So that the actual call frame header appears to return to the arity check fail return
 243     // PC, and that then returns to the register preservation thunk. The arity check thunk that
 244     // we return to will have the padding size encoded into it. It will then know to return
 245     // into the register preservation thunk, which uses the argument count to figure out where
 246     // registers are preserved.
 247
 248     // This code assumes that we're dealing with FunctionCode.
 249     RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
 250
 251     jit.add32(
 252         MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
 253         GPRInfo::regT3);
 254     MacroAssembler::Jump arityIntact = jit.branch32(
 255         MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
 256     jit.neg32(GPRInfo::regT3);
 257     jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
 258     jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
 259     jit.add32(GPRInfo::regT3, GPRInfo::regT2);
 260     arityIntact.link(&jit);
 261
 262     // First set up SP so that our data doesn't get clobbered by signals.
 263     unsigned conservativeStackDelta =
 264         registerPreservationOffset() +
 265         exit.m_values.numberOfLocals() * sizeof(Register) +
 266         maxFrameExtentForSlowPathCall;
 267     conservativeStackDelta = WTF::roundUpToMultipleOf(
 268         stackAlignmentBytes(), conservativeStackDelta);
 269     jit.addPtr(
 270         MacroAssembler::TrustedImm32(-conservativeStackDelta),
 271         MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
 272     jit.checkStackPointerAlignment();
 273
 274     jit.subPtr(
 275         MacroAssembler::TrustedImm32(registerPreservationOffset()),
 276         MacroAssembler::framePointerRegister);
 277
 278     // Copy the old frame data into its new location.
 279     jit.add32(MacroAssembler::TrustedImm32(JSStack::CallFrameHeaderSize), GPRInfo::regT2);
 280     jit.move(MacroAssembler::framePointerRegister, GPRInfo::regT1);
 281     MacroAssembler::Label loop = jit.label();
 282     jit.sub32(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
 283     jit.load64(MacroAssembler::Address(GPRInfo::regT1, offset), GPRInfo::regT0);
 284     jit.store64(GPRInfo::regT0, GPRInfo::regT1);
 285     jit.addPtr(MacroAssembler::TrustedImm32(sizeof(Register)), GPRInfo::regT1);
 286     jit.branchTest32(MacroAssembler::NonZero, GPRInfo::regT2).linkTo(loop, &jit);
 287
 288     // At this point regT1 points to where we would save our registers. Save them here.
 289     ptrdiff_t currentOffset = 0;
 290     for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
 291         if (!toSave.get(reg))
 292             continue;
 293         currentOffset += sizeof(Register);
 294         unsigned unwindIndex = jitCode->unwindInfo.indexOf(reg);
 295         if (unwindIndex == UINT_MAX) {
 296             // The FTL compilation didn't preserve this register. This means that it also
 297             // didn't use the register. So its value at the beginning of OSR exit should be
 298             // preserved by the thunk. Luckily, we saved all registers into the register
 299             // scratch buffer, so we can restore them from there.
 300             jit.load64(registerScratch + offsetOfReg(reg), GPRInfo::regT0);
 301         } else {
 302             // The FTL compilation preserved the register. Its new value is therefore
 303             // irrelevant, but we can get the value that was preserved by using the unwind
 304             // data. We've already copied all unwind-able preserved registers into the unwind
 305             // scratch buffer, so we can get it from there.
 306             jit.load64(unwindScratch + unwindIndex, GPRInfo::regT0);
 307         }
 308         jit.store64(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT1, currentOffset));
 309     }
 310
 311     // We need to make sure that we return into the register restoration thunk. This works
 312     // differently depending on whether or not we had arity issues.
 313     MacroAssembler::Jump arityIntactForReturnPC = jit.branch32(
 314         MacroAssembler::GreaterThanOrEqual,
 315         CCallHelpers::payloadFor(JSStack::ArgumentCount),
 316         MacroAssembler::TrustedImm32(codeBlock->numParameters()));
 317
 318     // The return PC in the call frame header points at exactly the right arity restoration
 319     // thunk. We don't want to change that. But the arity restoration thunk's frame has a
 320     // return PC and we want to reroute that to our register restoration thunk. The arity
 321     // restoration's return PC just just below regT1, and the register restoration's return PC
 322     // is right at regT1.
 323     jit.loadPtr(MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))), GPRInfo::regT0);
 324     jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
 325     jit.storePtr(
 326         MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
 327         MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))));
 328
 329     MacroAssembler::Jump arityReturnPCReady = jit.jump();
 330
 331     arityIntactForReturnPC.link(&jit);
 332
 333     jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), GPRInfo::regT0);
 334     jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
 335     jit.storePtr(
 336         MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
 337         MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()));
 338
 339     arityReturnPCReady.link(&jit);
 340
 341     // Now get state out of the scratch buffer and place it back into the stack. This part does
 342     // all reboxing.
 343     for (unsigned index = exit.m_values.size(); index--;) {
 344         int operand = exit.m_values.operandForIndex(index);
 345         ExitValue value = exit.m_values[index];
 346
 347         jit.load64(scratch + index, GPRInfo::regT0);
 348         reboxAccordingToFormat(
 349             value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
 350         jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(static_cast<VirtualRegister>(operand)));
 351     }
 352
 353     handleExitCounts(jit, exit);
 354     reifyInlinedCallFrames(jit, exit);
 355
 356     ArgumentsRecoveryGenerator argumentsRecovery;
 357     for (unsigned index = exit.m_values.size(); index--;) {
 358         if (!exit.m_values[index].isArgumentsObjectThatWasNotCreated())
 359             continue;
 360         int operand = exit.m_values.operandForIndex(index);
 361         argumentsRecovery.generateFor(operand, exit.m_codeOrigin, jit);
 362     }
 363
 364     adjustAndJumpToTarget(jit, exit);
 365
 366     LinkBuffer patchBuffer(*vm, jit, codeBlock);
 367     exit.m_code = FINALIZE_CODE_IF(
 368         shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
 369         patchBuffer,
 370         ("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
 371             exitID, toCString(exit.m_codeOrigin).data(),
 372             exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
 373             toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
 374             toCString(*record).data()));
 375 }
 376
 377 extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID)
 378 {
 379     SamplingRegion samplingRegion("FTL OSR Exit Compilation");
 380
 381     if (shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit())
 382         dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
 383
 384     CodeBlock* codeBlock = exec->codeBlock();
 385
 386     ASSERT(codeBlock);
 387     ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
 388
 389     VM* vm = &exec->vm();
 390
 391     // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
 392     // really be profitable.
 393     DeferGCForAWhile deferGC(vm->heap);
 394
 395     JITCode* jitCode = codeBlock->jitCode()->ftl();
 396     OSRExit& exit = jitCode->osrExit[exitID];
 397
 398     prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
 399
 400     compileStub(exitID, jitCode, exit, vm, codeBlock);
 401
 402     RepatchBuffer repatchBuffer(codeBlock);
 403     repatchBuffer.relink(
 404         exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
 405
 406     return exit.m_code.code().executableAddress();
 407 }
 408
 409 } } // namespace JSC::FTL
 410
 411 #endif // ENABLE(FTL_JIT)
 412