dfg/DFGSpeculativeJIT.h

   1 /*
   2  * Copyright (C) 2011-2015 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #ifndef DFGSpeculativeJIT_h
  27 #define DFGSpeculativeJIT_h
  28
  29 #if ENABLE(DFG_JIT)
  30
  31 #include "DFGAbstractInterpreter.h"
  32 #include "DFGGenerationInfo.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "DFGJITCompiler.h"
  35 #include "DFGOSRExit.h"
  36 #include "DFGOSRExitJumpPlaceholder.h"
  37 #include "DFGSilentRegisterSavePlan.h"
  38 #include "DFGValueSource.h"
  39 #include "JITOperations.h"
  40 #include "MarkedAllocator.h"
  41 #include "PutKind.h"
  42 #include "ValueRecovery.h"
  43 #include "VirtualRegister.h"
  44
  45 namespace JSC { namespace DFG {
  46
  47 class GPRTemporary;
  48 class JSValueOperand;
  49 class SlowPathGenerator;
  50 class SpeculativeJIT;
  51 class SpeculateInt32Operand;
  52 class SpeculateStrictInt32Operand;
  53 class SpeculateDoubleOperand;
  54 class SpeculateCellOperand;
  55 class SpeculateBooleanOperand;
  56
  57 enum GeneratedOperandType { GeneratedOperandTypeUnknown, GeneratedOperandInteger, GeneratedOperandJSValue};
  58
  59 inline GPRReg extractResult(GPRReg result) { return result; }
  60 #if USE(JSVALUE64)
  61 inline GPRReg extractResult(JSValueRegs result) { return result.gpr(); }
  62 #else
  63 inline JSValueRegs extractResult(JSValueRegs result) { return result; }
  64 #endif
  65 inline NoResultTag extractResult(NoResultTag) { return NoResult; }
  66
  67 // === SpeculativeJIT ===
  68 //
  69 // The SpeculativeJIT is used to generate a fast, but potentially
  70 // incomplete code path for the dataflow. When code generating
  71 // we may make assumptions about operand types, dynamically check,
  72 // and bail-out to an alternate code path if these checks fail.
  73 // Importantly, the speculative code path cannot be reentered once
  74 // a speculative check has failed. This allows the SpeculativeJIT
  75 // to propagate type information (including information that has
  76 // only speculatively been asserted) through the dataflow.
  77 class SpeculativeJIT {
  78     WTF_MAKE_FAST_ALLOCATED;
  79
  80     friend struct OSRExit;
  81 private:
  82     typedef JITCompiler::TrustedImm32 TrustedImm32;
  83     typedef JITCompiler::Imm32 Imm32;
  84     typedef JITCompiler::TrustedImmPtr TrustedImmPtr;
  85     typedef JITCompiler::ImmPtr ImmPtr;
  86     typedef JITCompiler::TrustedImm64 TrustedImm64;
  87     typedef JITCompiler::Imm64 Imm64;
  88
  89     // These constants are used to set priorities for spill order for
  90     // the register allocator.
  91 #if USE(JSVALUE64)
  92     enum SpillOrder {
  93         SpillOrderConstant = 1, // no spill, and cheap fill
  94         SpillOrderSpilled  = 2, // no spill
  95         SpillOrderJS       = 4, // needs spill
  96         SpillOrderCell     = 4, // needs spill
  97         SpillOrderStorage  = 4, // needs spill
  98         SpillOrderInteger  = 5, // needs spill and box
  99         SpillOrderBoolean  = 5, // needs spill and box
 100         SpillOrderDouble   = 6, // needs spill and convert
 101     };
 102 #elif USE(JSVALUE32_64)
 103     enum SpillOrder {
 104         SpillOrderConstant = 1, // no spill, and cheap fill
 105         SpillOrderSpilled  = 2, // no spill
 106         SpillOrderJS       = 4, // needs spill
 107         SpillOrderStorage  = 4, // needs spill
 108         SpillOrderDouble   = 4, // needs spill
 109         SpillOrderInteger  = 5, // needs spill and box
 110         SpillOrderCell     = 5, // needs spill and box
 111         SpillOrderBoolean  = 5, // needs spill and box
 112     };
 113 #endif
 114
 115     enum UseChildrenMode { CallUseChildren, UseChildrenCalledExplicitly };
 116
 117 public:
 118     SpeculativeJIT(JITCompiler&);
 119     ~SpeculativeJIT();
 120
 121     bool compile();
 122
 123     void createOSREntries();
 124     void linkOSREntries(LinkBuffer&);
 125
 126     BasicBlock* nextBlock()
 127     {
 128         for (BlockIndex resultIndex = m_block->index + 1; ; resultIndex++) {
 129             if (resultIndex >= m_jit.graph().numBlocks())
 130                 return 0;
 131             if (BasicBlock* result = m_jit.graph().block(resultIndex))
 132                 return result;
 133         }
 134     }
 135
 136 #if USE(JSVALUE64)
 137     GPRReg fillJSValue(Edge);
 138 #elif USE(JSVALUE32_64)
 139     bool fillJSValue(Edge, GPRReg&, GPRReg&, FPRReg&);
 140 #endif
 141     GPRReg fillStorage(Edge);
 142
 143     // lock and unlock GPR & FPR registers.
 144     void lock(GPRReg reg)
 145     {
 146         m_gprs.lock(reg);
 147     }
 148     void lock(FPRReg reg)
 149     {
 150         m_fprs.lock(reg);
 151     }
 152     void unlock(GPRReg reg)
 153     {
 154         m_gprs.unlock(reg);
 155     }
 156     void unlock(FPRReg reg)
 157     {
 158         m_fprs.unlock(reg);
 159     }
 160
 161     // Used to check whether a child node is on its last use,
 162     // and its machine registers may be reused.
 163     bool canReuse(Node* node)
 164     {
 165         return generationInfo(node).canReuse();
 166     }
 167     bool canReuse(Edge nodeUse)
 168     {
 169         return canReuse(nodeUse.node());
 170     }
 171     GPRReg reuse(GPRReg reg)
 172     {
 173         m_gprs.lock(reg);
 174         return reg;
 175     }
 176     FPRReg reuse(FPRReg reg)
 177     {
 178         m_fprs.lock(reg);
 179         return reg;
 180     }
 181
 182     // Allocate a gpr/fpr.
 183     GPRReg allocate()
 184     {
 185 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
 186         m_jit.addRegisterAllocationAtOffset(m_jit.debugOffset());
 187 #endif
 188         VirtualRegister spillMe;
 189         GPRReg gpr = m_gprs.allocate(spillMe);
 190         if (spillMe.isValid()) {
 191 #if USE(JSVALUE32_64)
 192             GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
 193             if ((info.registerFormat() & DataFormatJS))
 194                 m_gprs.release(info.tagGPR() == gpr ? info.payloadGPR() : info.tagGPR());
 195 #endif
 196             spill(spillMe);
 197         }
 198         return gpr;
 199     }
 200     GPRReg allocate(GPRReg specific)
 201     {
 202 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
 203         m_jit.addRegisterAllocationAtOffset(m_jit.debugOffset());
 204 #endif
 205         VirtualRegister spillMe = m_gprs.allocateSpecific(specific);
 206         if (spillMe.isValid()) {
 207 #if USE(JSVALUE32_64)
 208             GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
 209             RELEASE_ASSERT(info.registerFormat() != DataFormatJSDouble);
 210             if ((info.registerFormat() & DataFormatJS))
 211                 m_gprs.release(info.tagGPR() == specific ? info.payloadGPR() : info.tagGPR());
 212 #endif
 213             spill(spillMe);
 214         }
 215         return specific;
 216     }
 217     GPRReg tryAllocate()
 218     {
 219         return m_gprs.tryAllocate();
 220     }
 221     FPRReg fprAllocate()
 222     {
 223 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
 224         m_jit.addRegisterAllocationAtOffset(m_jit.debugOffset());
 225 #endif
 226         VirtualRegister spillMe;
 227         FPRReg fpr = m_fprs.allocate(spillMe);
 228         if (spillMe.isValid())
 229             spill(spillMe);
 230         return fpr;
 231     }
 232
 233     // Check whether a VirtualRegsiter is currently in a machine register.
 234     // We use this when filling operands to fill those that are already in
 235     // machine registers first (by locking VirtualRegsiters that are already
 236     // in machine register before filling those that are not we attempt to
 237     // avoid spilling values we will need immediately).
 238     bool isFilled(Node* node)
 239     {
 240         return generationInfo(node).registerFormat() != DataFormatNone;
 241     }
 242     bool isFilledDouble(Node* node)
 243     {
 244         return generationInfo(node).registerFormat() == DataFormatDouble;
 245     }
 246
 247     // Called on an operand once it has been consumed by a parent node.
 248     void use(Node* node)
 249     {
 250         if (!node->hasResult())
 251             return;
 252         GenerationInfo& info = generationInfo(node);
 253
 254         // use() returns true when the value becomes dead, and any
 255         // associated resources may be freed.
 256         if (!info.use(*m_stream))
 257             return;
 258
 259         // Release the associated machine registers.
 260         DataFormat registerFormat = info.registerFormat();
 261 #if USE(JSVALUE64)
 262         if (registerFormat == DataFormatDouble)
 263             m_fprs.release(info.fpr());
 264         else if (registerFormat != DataFormatNone)
 265             m_gprs.release(info.gpr());
 266 #elif USE(JSVALUE32_64)
 267         if (registerFormat == DataFormatDouble)
 268             m_fprs.release(info.fpr());
 269         else if (registerFormat & DataFormatJS) {
 270             m_gprs.release(info.tagGPR());
 271             m_gprs.release(info.payloadGPR());
 272         } else if (registerFormat != DataFormatNone)
 273             m_gprs.release(info.gpr());
 274 #endif
 275     }
 276     void use(Edge nodeUse)
 277     {
 278         use(nodeUse.node());
 279     }
 280
 281     RegisterSet usedRegisters();
 282
 283     bool masqueradesAsUndefinedWatchpointIsStillValid(const CodeOrigin& codeOrigin)
 284     {
 285         return m_jit.graph().masqueradesAsUndefinedWatchpointIsStillValid(codeOrigin);
 286     }
 287     bool masqueradesAsUndefinedWatchpointIsStillValid()
 288     {
 289         return masqueradesAsUndefinedWatchpointIsStillValid(m_currentNode->origin.semantic);
 290     }
 291
 292 #if ENABLE(GGC)
 293     void storeToWriteBarrierBuffer(GPRReg cell, GPRReg scratch1, GPRReg scratch2);
 294
 295     void writeBarrier(GPRReg owner, GPRReg scratch1, GPRReg scratch2);
 296
 297     void writeBarrier(GPRReg owner, GPRReg value, Edge valueUse, GPRReg scratch1, GPRReg scratch2);
 298 #endif
 299     void compileStoreBarrier(Node*);
 300
 301     static GPRReg selectScratchGPR(GPRReg preserve1 = InvalidGPRReg, GPRReg preserve2 = InvalidGPRReg, GPRReg preserve3 = InvalidGPRReg, GPRReg preserve4 = InvalidGPRReg)
 302     {
 303         return AssemblyHelpers::selectScratchGPR(preserve1, preserve2, preserve3, preserve4);
 304     }
 305
 306     // Called by the speculative operand types, below, to fill operand to
 307     // machine registers, implicitly generating speculation checks as needed.
 308     GPRReg fillSpeculateInt32(Edge, DataFormat& returnFormat);
 309     GPRReg fillSpeculateInt32Strict(Edge);
 310     GPRReg fillSpeculateInt52(Edge, DataFormat desiredFormat);
 311     FPRReg fillSpeculateDouble(Edge);
 312     GPRReg fillSpeculateCell(Edge);
 313     GPRReg fillSpeculateBoolean(Edge);
 314     GeneratedOperandType checkGeneratedTypeForToInt32(Node*);
 315
 316     void addSlowPathGenerator(std::unique_ptr<SlowPathGenerator>);
 317     void runSlowPathGenerators();
 318
 319     void compile(Node*);
 320     void noticeOSRBirth(Node*);
 321     void bail(AbortReason);
 322     void compileCurrentBlock();
 323
 324     void checkArgumentTypes();
 325
 326     void clearGenerationInfo();
 327
 328     // These methods are used when generating 'unexpected'
 329     // calls out from JIT code to C++ helper routines -
 330     // they spill all live values to the appropriate
 331     // slots in the JSStack without changing any state
 332     // in the GenerationInfo.
 333     SilentRegisterSavePlan silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source);
 334     SilentRegisterSavePlan silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source);
 335     void silentSpill(const SilentRegisterSavePlan&);
 336     void silentFill(const SilentRegisterSavePlan&, GPRReg canTrample);
 337
 338     template<typename CollectionType>
 339     void silentSpillAllRegistersImpl(bool doSpill, CollectionType& plans, GPRReg exclude, GPRReg exclude2 = InvalidGPRReg, FPRReg fprExclude = InvalidFPRReg)
 340     {
 341         ASSERT(plans.isEmpty());
 342         for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {
 343             GPRReg gpr = iter.regID();
 344             if (iter.name().isValid() && gpr != exclude && gpr != exclude2) {
 345                 SilentRegisterSavePlan plan = silentSavePlanForGPR(iter.name(), gpr);
 346                 if (doSpill)
 347                     silentSpill(plan);
 348                 plans.append(plan);
 349             }
 350         }
 351         for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {
 352             if (iter.name().isValid() && iter.regID() != fprExclude) {
 353                 SilentRegisterSavePlan plan = silentSavePlanForFPR(iter.name(), iter.regID());
 354                 if (doSpill)
 355                     silentSpill(plan);
 356                 plans.append(plan);
 357             }
 358         }
 359     }
 360     template<typename CollectionType>
 361     void silentSpillAllRegistersImpl(bool doSpill, CollectionType& plans, NoResultTag)
 362     {
 363         silentSpillAllRegistersImpl(doSpill, plans, InvalidGPRReg, InvalidGPRReg, InvalidFPRReg);
 364     }
 365     template<typename CollectionType>
 366     void silentSpillAllRegistersImpl(bool doSpill, CollectionType& plans, FPRReg exclude)
 367     {
 368         silentSpillAllRegistersImpl(doSpill, plans, InvalidGPRReg, InvalidGPRReg, exclude);
 369     }
 370 #if USE(JSVALUE32_64)
 371     template<typename CollectionType>
 372     void silentSpillAllRegistersImpl(bool doSpill, CollectionType& plans, JSValueRegs exclude)
 373     {
 374         silentSpillAllRegistersImpl(doSpill, plans, exclude.tagGPR(), exclude.payloadGPR());
 375     }
 376 #endif
 377
 378     void silentSpillAllRegisters(GPRReg exclude, GPRReg exclude2 = InvalidGPRReg, FPRReg fprExclude = InvalidFPRReg)
 379     {
 380         silentSpillAllRegistersImpl(true, m_plans, exclude, exclude2, fprExclude);
 381     }
 382     void silentSpillAllRegisters(FPRReg exclude)
 383     {
 384         silentSpillAllRegisters(InvalidGPRReg, InvalidGPRReg, exclude);
 385     }
 386
 387     static GPRReg pickCanTrample(GPRReg exclude)
 388     {
 389         GPRReg result = GPRInfo::regT0;
 390         if (result == exclude)
 391             result = GPRInfo::regT1;
 392         return result;
 393     }
 394     static GPRReg pickCanTrample(FPRReg)
 395     {
 396         return GPRInfo::regT0;
 397     }
 398     static GPRReg pickCanTrample(NoResultTag)
 399     {
 400         return GPRInfo::regT0;
 401     }
 402
 403 #if USE(JSVALUE32_64)
 404     static GPRReg pickCanTrample(JSValueRegs exclude)
 405     {
 406         GPRReg result = GPRInfo::regT0;
 407         if (result == exclude.tagGPR()) {
 408             result = GPRInfo::regT1;
 409             if (result == exclude.payloadGPR())
 410                 result = GPRInfo::regT2;
 411         } else if (result == exclude.payloadGPR()) {
 412             result = GPRInfo::regT1;
 413             if (result == exclude.tagGPR())
 414                 result = GPRInfo::regT2;
 415         }
 416         return result;
 417     }
 418 #endif
 419
 420     template<typename RegisterType>
 421     void silentFillAllRegisters(RegisterType exclude)
 422     {
 423         GPRReg canTrample = pickCanTrample(exclude);
 424
 425         while (!m_plans.isEmpty()) {
 426             SilentRegisterSavePlan& plan = m_plans.last();
 427             silentFill(plan, canTrample);
 428             m_plans.removeLast();
 429         }
 430     }
 431
 432     // These methods convert between doubles, and doubles boxed and JSValues.
 433 #if USE(JSVALUE64)
 434     GPRReg boxDouble(FPRReg fpr, GPRReg gpr)
 435     {
 436         return m_jit.boxDouble(fpr, gpr);
 437     }
 438     FPRReg unboxDouble(GPRReg gpr, FPRReg fpr)
 439     {
 440         return m_jit.unboxDouble(gpr, fpr);
 441     }
 442     GPRReg boxDouble(FPRReg fpr)
 443     {
 444         return boxDouble(fpr, allocate());
 445     }
 446
 447     void boxInt52(GPRReg sourceGPR, GPRReg targetGPR, DataFormat);
 448 #elif USE(JSVALUE32_64)
 449     void boxDouble(FPRReg fpr, GPRReg tagGPR, GPRReg payloadGPR)
 450     {
 451         m_jit.boxDouble(fpr, tagGPR, payloadGPR);
 452     }
 453     void unboxDouble(GPRReg tagGPR, GPRReg payloadGPR, FPRReg fpr, FPRReg scratchFPR)
 454     {
 455         m_jit.unboxDouble(tagGPR, payloadGPR, fpr, scratchFPR);
 456     }
 457 #endif
 458     void boxDouble(FPRReg fpr, JSValueRegs regs)
 459     {
 460         m_jit.boxDouble(fpr, regs);
 461     }
 462
 463     // Spill a VirtualRegister to the JSStack.
 464     void spill(VirtualRegister spillMe)
 465     {
 466         GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
 467
 468 #if USE(JSVALUE32_64)
 469         if (info.registerFormat() == DataFormatNone) // it has been spilled. JS values which have two GPRs can reach here
 470             return;
 471 #endif
 472         // Check the GenerationInfo to see if this value need writing
 473         // to the JSStack - if not, mark it as spilled & return.
 474         if (!info.needsSpill()) {
 475             info.setSpilled(*m_stream, spillMe);
 476             return;
 477         }
 478
 479         DataFormat spillFormat = info.registerFormat();
 480         switch (spillFormat) {
 481         case DataFormatStorage: {
 482             // This is special, since it's not a JS value - as in it's not visible to JS
 483             // code.
 484             m_jit.storePtr(info.gpr(), JITCompiler::addressFor(spillMe));
 485             info.spill(*m_stream, spillMe, DataFormatStorage);
 486             return;
 487         }
 488
 489         case DataFormatInt32: {
 490             m_jit.store32(info.gpr(), JITCompiler::payloadFor(spillMe));
 491             info.spill(*m_stream, spillMe, DataFormatInt32);
 492             return;
 493         }
 494
 495 #if USE(JSVALUE64)
 496         case DataFormatDouble: {
 497             m_jit.storeDouble(info.fpr(), JITCompiler::addressFor(spillMe));
 498             info.spill(*m_stream, spillMe, DataFormatDouble);
 499             return;
 500         }
 501
 502         case DataFormatInt52:
 503         case DataFormatStrictInt52: {
 504             m_jit.store64(info.gpr(), JITCompiler::addressFor(spillMe));
 505             info.spill(*m_stream, spillMe, spillFormat);
 506             return;
 507         }
 508
 509         default:
 510             // The following code handles JSValues, int32s, and cells.
 511             RELEASE_ASSERT(spillFormat == DataFormatCell || spillFormat & DataFormatJS);
 512
 513             GPRReg reg = info.gpr();
 514             // We need to box int32 and cell values ...
 515             // but on JSVALUE64 boxing a cell is a no-op!
 516             if (spillFormat == DataFormatInt32)
 517                 m_jit.or64(GPRInfo::tagTypeNumberRegister, reg);
 518
 519             // Spill the value, and record it as spilled in its boxed form.
 520             m_jit.store64(reg, JITCompiler::addressFor(spillMe));
 521             info.spill(*m_stream, spillMe, (DataFormat)(spillFormat | DataFormatJS));
 522             return;
 523 #elif USE(JSVALUE32_64)
 524         case DataFormatCell:
 525         case DataFormatBoolean: {
 526             m_jit.store32(info.gpr(), JITCompiler::payloadFor(spillMe));
 527             info.spill(*m_stream, spillMe, spillFormat);
 528             return;
 529         }
 530
 531         case DataFormatDouble: {
 532             // On JSVALUE32_64 boxing a double is a no-op.
 533             m_jit.storeDouble(info.fpr(), JITCompiler::addressFor(spillMe));
 534             info.spill(*m_stream, spillMe, DataFormatDouble);
 535             return;
 536         }
 537
 538         default:
 539             // The following code handles JSValues.
 540             RELEASE_ASSERT(spillFormat & DataFormatJS);
 541             m_jit.store32(info.tagGPR(), JITCompiler::tagFor(spillMe));
 542             m_jit.store32(info.payloadGPR(), JITCompiler::payloadFor(spillMe));
 543             info.spill(*m_stream, spillMe, spillFormat);
 544             return;
 545 #endif
 546         }
 547     }
 548
 549     bool isKnownInteger(Node* node) { return m_state.forNode(node).isType(SpecInt32); }
 550     bool isKnownCell(Node* node) { return m_state.forNode(node).isType(SpecCell); }
 551
 552     bool isKnownNotInteger(Node* node) { return !(m_state.forNode(node).m_type & SpecInt32); }
 553     bool isKnownNotNumber(Node* node) { return !(m_state.forNode(node).m_type & SpecFullNumber); }
 554     bool isKnownNotCell(Node* node) { return !(m_state.forNode(node).m_type & SpecCell); }
 555
 556     UniquedStringImpl* identifierUID(unsigned index)
 557     {
 558         return m_jit.graph().identifiers()[index];
 559     }
 560
 561     // Spill all VirtualRegisters back to the JSStack.
 562     void flushRegisters()
 563     {
 564         for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {
 565             if (iter.name().isValid()) {
 566                 spill(iter.name());
 567                 iter.release();
 568             }
 569         }
 570         for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {
 571             if (iter.name().isValid()) {
 572                 spill(iter.name());
 573                 iter.release();
 574             }
 575         }
 576     }
 577
 578     // Used to ASSERT flushRegisters() has been called prior to
 579     // calling out from JIT code to a C helper function.
 580     bool isFlushed()
 581     {
 582         for (gpr_iterator iter = m_gprs.begin(); iter != m_gprs.end(); ++iter) {
 583             if (iter.name().isValid())
 584                 return false;
 585         }
 586         for (fpr_iterator iter = m_fprs.begin(); iter != m_fprs.end(); ++iter) {
 587             if (iter.name().isValid())
 588                 return false;
 589         }
 590         return true;
 591     }
 592
 593 #if USE(JSVALUE64)
 594     static MacroAssembler::Imm64 valueOfJSConstantAsImm64(Node* node)
 595     {
 596         return MacroAssembler::Imm64(JSValue::encode(node->asJSValue()));
 597     }
 598 #endif
 599
 600     // Helper functions to enable code sharing in implementations of bit/shift ops.
 601     void bitOp(NodeType op, int32_t imm, GPRReg op1, GPRReg result)
 602     {
 603         switch (op) {
 604         case BitAnd:
 605             m_jit.and32(Imm32(imm), op1, result);
 606             break;
 607         case BitOr:
 608             m_jit.or32(Imm32(imm), op1, result);
 609             break;
 610         case BitXor:
 611             m_jit.xor32(Imm32(imm), op1, result);
 612             break;
 613         default:
 614             RELEASE_ASSERT_NOT_REACHED();
 615         }
 616     }
 617     void bitOp(NodeType op, GPRReg op1, GPRReg op2, GPRReg result)
 618     {
 619         switch (op) {
 620         case BitAnd:
 621             m_jit.and32(op1, op2, result);
 622             break;
 623         case BitOr:
 624             m_jit.or32(op1, op2, result);
 625             break;
 626         case BitXor:
 627             m_jit.xor32(op1, op2, result);
 628             break;
 629         default:
 630             RELEASE_ASSERT_NOT_REACHED();
 631         }
 632     }
 633     void shiftOp(NodeType op, GPRReg op1, int32_t shiftAmount, GPRReg result)
 634     {
 635         switch (op) {
 636         case BitRShift:
 637             m_jit.rshift32(op1, Imm32(shiftAmount), result);
 638             break;
 639         case BitLShift:
 640             m_jit.lshift32(op1, Imm32(shiftAmount), result);
 641             break;
 642         case BitURShift:
 643             m_jit.urshift32(op1, Imm32(shiftAmount), result);
 644             break;
 645         default:
 646             RELEASE_ASSERT_NOT_REACHED();
 647         }
 648     }
 649     void shiftOp(NodeType op, GPRReg op1, GPRReg shiftAmount, GPRReg result)
 650     {
 651         switch (op) {
 652         case BitRShift:
 653             m_jit.rshift32(op1, shiftAmount, result);
 654             break;
 655         case BitLShift:
 656             m_jit.lshift32(op1, shiftAmount, result);
 657             break;
 658         case BitURShift:
 659             m_jit.urshift32(op1, shiftAmount, result);
 660             break;
 661         default:
 662             RELEASE_ASSERT_NOT_REACHED();
 663         }
 664     }
 665
 666     // Returns the index of the branch node if peephole is okay, UINT_MAX otherwise.
 667     unsigned detectPeepHoleBranch()
 668     {
 669         // Check that no intervening nodes will be generated.
 670         for (unsigned index = m_indexInBlock + 1; index < m_block->size() - 1; ++index) {
 671             Node* node = m_block->at(index);
 672             if (!node->shouldGenerate())
 673                 continue;
 674             // Check if it's a Phantom that can be safely ignored.
 675             if (node->op() == Phantom && !node->child1())
 676                 continue;
 677             return UINT_MAX;
 678         }
 679
 680         // Check if the lastNode is a branch on this node.
 681         Node* lastNode = m_block->terminal();
 682         return lastNode->op() == Branch && lastNode->child1() == m_currentNode ? m_block->size() - 1 : UINT_MAX;
 683     }
 684
 685     void compileMovHint(Node*);
 686     void compileMovHintAndCheck(Node*);
 687
 688 #if USE(JSVALUE64)
 689     void cachedGetById(CodeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget = JITCompiler::Jump(), SpillRegistersMode = NeedToSpill);
 690     void cachedPutById(CodeOrigin, GPRReg base, GPRReg value, GPRReg scratchGPR, unsigned identifierNumber, PutKind, JITCompiler::Jump slowPathTarget = JITCompiler::Jump(), SpillRegistersMode = NeedToSpill);
 691 #elif USE(JSVALUE32_64)
 692     void cachedGetById(CodeOrigin, GPRReg baseTagGPROrNone, GPRReg basePayloadGPR, GPRReg resultTagGPR, GPRReg resultPayloadGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget = JITCompiler::Jump(), SpillRegistersMode = NeedToSpill);
 693     void cachedPutById(CodeOrigin, GPRReg basePayloadGPR, GPRReg valueTagGPR, GPRReg valuePayloadGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind, JITCompiler::Jump slowPathTarget = JITCompiler::Jump(), SpillRegistersMode = NeedToSpill);
 694 #endif
 695
 696     void compileIn(Node*);
 697
 698     void compileBaseValueStoreBarrier(Edge& baseEdge, Edge& valueEdge);
 699
 700     void nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert = false);
 701     void nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert = false);
 702     bool nonSpeculativeCompareNull(Node*, Edge operand, bool invert = false);
 703
 704     void nonSpeculativePeepholeBranch(Node*, Node* branchNode, MacroAssembler::RelationalCondition, S_JITOperation_EJJ helperFunction);
 705     void nonSpeculativeNonPeepholeCompare(Node*, MacroAssembler::RelationalCondition, S_JITOperation_EJJ helperFunction);
 706     bool nonSpeculativeCompare(Node*, MacroAssembler::RelationalCondition, S_JITOperation_EJJ helperFunction);
 707
 708     void nonSpeculativePeepholeStrictEq(Node*, Node* branchNode, bool invert = false);
 709     void nonSpeculativeNonPeepholeStrictEq(Node*, bool invert = false);
 710     bool nonSpeculativeStrictEq(Node*, bool invert = false);
 711
 712     void compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchAndResultReg, GPRReg scratch2Reg);
 713     void compileInstanceOf(Node*);
 714
 715     void emitCall(Node*);
 716
 717     // Called once a node has completed code generation but prior to setting
 718     // its result, to free up its children. (This must happen prior to setting
 719     // the nodes result, since the node may have the same VirtualRegister as
 720     // a child, and as such will use the same GeneratioInfo).
 721     void useChildren(Node*);
 722
 723     // These method called to initialize the the GenerationInfo
 724     // to describe the result of an operation.
 725     void int32Result(GPRReg reg, Node* node, DataFormat format = DataFormatInt32, UseChildrenMode mode = CallUseChildren)
 726     {
 727         if (mode == CallUseChildren)
 728             useChildren(node);
 729
 730         VirtualRegister virtualRegister = node->virtualRegister();
 731         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 732
 733         if (format == DataFormatInt32) {
 734             m_jit.jitAssertIsInt32(reg);
 735             m_gprs.retain(reg, virtualRegister, SpillOrderInteger);
 736             info.initInt32(node, node->refCount(), reg);
 737         } else {
 738 #if USE(JSVALUE64)
 739             RELEASE_ASSERT(format == DataFormatJSInt32);
 740             m_jit.jitAssertIsJSInt32(reg);
 741             m_gprs.retain(reg, virtualRegister, SpillOrderJS);
 742             info.initJSValue(node, node->refCount(), reg, format);
 743 #elif USE(JSVALUE32_64)
 744             RELEASE_ASSERT_NOT_REACHED();
 745 #endif
 746         }
 747     }
 748     void int32Result(GPRReg reg, Node* node, UseChildrenMode mode)
 749     {
 750         int32Result(reg, node, DataFormatInt32, mode);
 751     }
 752     void int52Result(GPRReg reg, Node* node, DataFormat format, UseChildrenMode mode = CallUseChildren)
 753     {
 754         if (mode == CallUseChildren)
 755             useChildren(node);
 756
 757         VirtualRegister virtualRegister = node->virtualRegister();
 758         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 759
 760         m_gprs.retain(reg, virtualRegister, SpillOrderJS);
 761         info.initInt52(node, node->refCount(), reg, format);
 762     }
 763     void int52Result(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 764     {
 765         int52Result(reg, node, DataFormatInt52, mode);
 766     }
 767     void strictInt52Result(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 768     {
 769         int52Result(reg, node, DataFormatStrictInt52, mode);
 770     }
 771     void noResult(Node* node, UseChildrenMode mode = CallUseChildren)
 772     {
 773         if (mode == UseChildrenCalledExplicitly)
 774             return;
 775         useChildren(node);
 776     }
 777     void cellResult(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 778     {
 779         if (mode == CallUseChildren)
 780             useChildren(node);
 781
 782         VirtualRegister virtualRegister = node->virtualRegister();
 783         m_gprs.retain(reg, virtualRegister, SpillOrderCell);
 784         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 785         info.initCell(node, node->refCount(), reg);
 786     }
 787     void blessedBooleanResult(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 788     {
 789 #if USE(JSVALUE64)
 790         jsValueResult(reg, node, DataFormatJSBoolean, mode);
 791 #else
 792         booleanResult(reg, node, mode);
 793 #endif
 794     }
 795     void unblessedBooleanResult(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 796     {
 797 #if USE(JSVALUE64)
 798         blessBoolean(reg);
 799 #endif
 800         blessedBooleanResult(reg, node, mode);
 801     }
 802 #if USE(JSVALUE64)
 803     void jsValueResult(GPRReg reg, Node* node, DataFormat format = DataFormatJS, UseChildrenMode mode = CallUseChildren)
 804     {
 805         if (format == DataFormatJSInt32)
 806             m_jit.jitAssertIsJSInt32(reg);
 807
 808         if (mode == CallUseChildren)
 809             useChildren(node);
 810
 811         VirtualRegister virtualRegister = node->virtualRegister();
 812         m_gprs.retain(reg, virtualRegister, SpillOrderJS);
 813         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 814         info.initJSValue(node, node->refCount(), reg, format);
 815     }
 816     void jsValueResult(GPRReg reg, Node* node, UseChildrenMode mode)
 817     {
 818         jsValueResult(reg, node, DataFormatJS, mode);
 819     }
 820 #elif USE(JSVALUE32_64)
 821     void booleanResult(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 822     {
 823         if (mode == CallUseChildren)
 824             useChildren(node);
 825
 826         VirtualRegister virtualRegister = node->virtualRegister();
 827         m_gprs.retain(reg, virtualRegister, SpillOrderBoolean);
 828         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 829         info.initBoolean(node, node->refCount(), reg);
 830     }
 831     void jsValueResult(GPRReg tag, GPRReg payload, Node* node, DataFormat format = DataFormatJS, UseChildrenMode mode = CallUseChildren)
 832     {
 833         if (mode == CallUseChildren)
 834             useChildren(node);
 835
 836         VirtualRegister virtualRegister = node->virtualRegister();
 837         m_gprs.retain(tag, virtualRegister, SpillOrderJS);
 838         m_gprs.retain(payload, virtualRegister, SpillOrderJS);
 839         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 840         info.initJSValue(node, node->refCount(), tag, payload, format);
 841     }
 842     void jsValueResult(GPRReg tag, GPRReg payload, Node* node, UseChildrenMode mode)
 843     {
 844         jsValueResult(tag, payload, node, DataFormatJS, mode);
 845     }
 846 #endif
 847     void jsValueResult(JSValueRegs regs, Node* node, DataFormat format = DataFormatJS, UseChildrenMode mode = CallUseChildren)
 848     {
 849 #if USE(JSVALUE64)
 850         jsValueResult(regs.gpr(), node, format, mode);
 851 #else
 852         jsValueResult(regs.tagGPR(), regs.payloadGPR(), node, format, mode);
 853 #endif
 854     }
 855     void storageResult(GPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 856     {
 857         if (mode == CallUseChildren)
 858             useChildren(node);
 859
 860         VirtualRegister virtualRegister = node->virtualRegister();
 861         m_gprs.retain(reg, virtualRegister, SpillOrderStorage);
 862         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 863         info.initStorage(node, node->refCount(), reg);
 864     }
 865     void doubleResult(FPRReg reg, Node* node, UseChildrenMode mode = CallUseChildren)
 866     {
 867         if (mode == CallUseChildren)
 868             useChildren(node);
 869
 870         VirtualRegister virtualRegister = node->virtualRegister();
 871         m_fprs.retain(reg, virtualRegister, SpillOrderDouble);
 872         GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
 873         info.initDouble(node, node->refCount(), reg);
 874     }
 875     void initConstantInfo(Node* node)
 876     {
 877         ASSERT(node->hasConstant());
 878         generationInfo(node).initConstant(node, node->refCount());
 879     }
 880
 881     // These methods add calls to C++ helper functions.
 882     // These methods are broadly value representation specific (i.e.
 883     // deal with the fact that a JSValue may be passed in one or two
 884     // machine registers, and delegate the calling convention specific
 885     // decision as to how to fill the regsiters to setupArguments* methods.
 886
 887     JITCompiler::Call callOperation(V_JITOperation_E operation)
 888     {
 889         m_jit.setupArgumentsExecState();
 890         return appendCallWithExceptionCheck(operation);
 891     }
 892     JITCompiler::Call callOperation(P_JITOperation_E operation, GPRReg result)
 893     {
 894         m_jit.setupArgumentsExecState();
 895         return appendCallWithExceptionCheckSetResult(operation, result);
 896     }
 897     JITCompiler::Call callOperation(P_JITOperation_EC operation, GPRReg result, GPRReg cell)
 898     {
 899         m_jit.setupArgumentsWithExecState(cell);
 900         return appendCallWithExceptionCheckSetResult(operation, result);
 901     }
 902     JITCompiler::Call callOperation(P_JITOperation_EO operation, GPRReg result, GPRReg object)
 903     {
 904         m_jit.setupArgumentsWithExecState(object);
 905         return appendCallWithExceptionCheckSetResult(operation, result);
 906     }
 907     JITCompiler::Call callOperation(P_JITOperation_EOS operation, GPRReg result, GPRReg object, size_t size)
 908     {
 909         m_jit.setupArgumentsWithExecState(object, TrustedImmPtr(size));
 910         return appendCallWithExceptionCheckSetResult(operation, result);
 911     }
 912     JITCompiler::Call callOperation(P_JITOperation_EOZ operation, GPRReg result, GPRReg object, int32_t size)
 913     {
 914         m_jit.setupArgumentsWithExecState(object, TrustedImmPtr(size));
 915         return appendCallWithExceptionCheckSetResult(operation, result);
 916     }
 917     JITCompiler::Call callOperation(C_JITOperation_EOZ operation, GPRReg result, GPRReg object, int32_t size)
 918     {
 919         m_jit.setupArgumentsWithExecState(object, TrustedImmPtr(static_cast<size_t>(size)));
 920         return appendCallWithExceptionCheckSetResult(operation, result);
 921     }
 922     JITCompiler::Call callOperation(P_JITOperation_EPS operation, GPRReg result, GPRReg old, size_t size)
 923     {
 924         m_jit.setupArgumentsWithExecState(old, TrustedImmPtr(size));
 925         return appendCallWithExceptionCheckSetResult(operation, result);
 926     }
 927     JITCompiler::Call callOperation(P_JITOperation_ES operation, GPRReg result, size_t size)
 928     {
 929         m_jit.setupArgumentsWithExecState(TrustedImmPtr(size));
 930         return appendCallWithExceptionCheckSetResult(operation, result);
 931     }
 932     JITCompiler::Call callOperation(P_JITOperation_ESJss operation, GPRReg result, size_t index, GPRReg arg1)
 933     {
 934         m_jit.setupArgumentsWithExecState(TrustedImmPtr(index), arg1);
 935         return appendCallWithExceptionCheckSetResult(operation, result);
 936     }
 937     JITCompiler::Call callOperation(P_JITOperation_ESt operation, GPRReg result, Structure* structure)
 938     {
 939         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure));
 940         return appendCallWithExceptionCheckSetResult(operation, result);
 941     }
 942     JITCompiler::Call callOperation(P_JITOperation_EStZ operation, GPRReg result, Structure* structure, GPRReg arg2)
 943     {
 944         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), arg2);
 945         return appendCallWithExceptionCheckSetResult(operation, result);
 946     }
 947     JITCompiler::Call callOperation(P_JITOperation_EStZ operation, GPRReg result, Structure* structure, size_t arg2)
 948     {
 949         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), TrustedImm32(arg2));
 950         return appendCallWithExceptionCheckSetResult(operation, result);
 951     }
 952     JITCompiler::Call callOperation(P_JITOperation_EStZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
 953     {
 954         m_jit.setupArgumentsWithExecState(arg1, arg2);
 955         return appendCallWithExceptionCheckSetResult(operation, result);
 956     }
 957     JITCompiler::Call callOperation(P_JITOperation_EStPS operation, GPRReg result, Structure* structure, void* pointer, size_t size)
 958     {
 959         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), TrustedImmPtr(pointer), TrustedImmPtr(size));
 960         return appendCallWithExceptionCheckSetResult(operation, result);
 961     }
 962     JITCompiler::Call callOperation(P_JITOperation_EStSS operation, GPRReg result, Structure* structure, size_t index, size_t size)
 963     {
 964         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), TrustedImmPtr(index), TrustedImmPtr(size));
 965         return appendCallWithExceptionCheckSetResult(operation, result);
 966     }
 967     JITCompiler::Call callOperation(C_JITOperation_E operation, GPRReg result)
 968     {
 969         m_jit.setupArgumentsExecState();
 970         return appendCallWithExceptionCheckSetResult(operation, result);
 971     }
 972     JITCompiler::Call callOperation(C_JITOperation_EC operation, GPRReg result, GPRReg arg1)
 973     {
 974         m_jit.setupArgumentsWithExecState(arg1);
 975         return appendCallWithExceptionCheckSetResult(operation, result);
 976     }
 977     JITCompiler::Call callOperation(C_JITOperation_EC operation, GPRReg result, JSCell* cell)
 978     {
 979         m_jit.setupArgumentsWithExecState(TrustedImmPtr(cell));
 980         return appendCallWithExceptionCheckSetResult(operation, result);
 981     }
 982     JITCompiler::Call callOperation(C_JITOperation_ECZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
 983     {
 984         m_jit.setupArgumentsWithExecState(arg1, arg2);
 985         return appendCallWithExceptionCheckSetResult(operation, result);
 986     }
 987     JITCompiler::Call callOperation(C_JITOperation_ECZC operation, GPRReg result, GPRReg arg1, GPRReg arg2, GPRReg arg3)
 988     {
 989         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
 990         return appendCallWithExceptionCheckSetResult(operation, result);
 991     }
 992     JITCompiler::Call callOperation(C_JITOperation_EJscC operation, GPRReg result, GPRReg arg1, JSCell* cell)
 993     {
 994         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(cell));
 995         return appendCallWithExceptionCheckSetResult(operation, result);
 996     }
 997     JITCompiler::Call callOperation(C_JITOperation_EIcf operation, GPRReg result, InlineCallFrame* inlineCallFrame)
 998     {
 999         m_jit.setupArgumentsWithExecState(TrustedImmPtr(inlineCallFrame));
1000         return appendCallWithExceptionCheckSetResult(operation, result);
1001     }
1002     JITCompiler::Call callOperation(C_JITOperation_ESt operation, GPRReg result, Structure* structure)
1003     {
1004         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure));
1005         return appendCallWithExceptionCheckSetResult(operation, result);
1006     }
1007     JITCompiler::Call callOperation(C_JITOperation_EStJscSymtab operation, GPRReg result, Structure* structure, GPRReg scope, SymbolTable* table)
1008     {
1009         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), scope, TrustedImmPtr(table));
1010         return appendCallWithExceptionCheckSetResult(operation, result);
1011     }
1012     JITCompiler::Call callOperation(C_JITOperation_EStZ operation, GPRReg result, Structure* structure, unsigned knownLength)
1013     {
1014         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), TrustedImm32(knownLength));
1015         return appendCallWithExceptionCheckSetResult(operation, result);
1016     }
1017     JITCompiler::Call callOperation(C_JITOperation_EStZZ operation, GPRReg result, Structure* structure, unsigned knownLength, unsigned minCapacity)
1018     {
1019         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), TrustedImm32(knownLength), TrustedImm32(minCapacity));
1020         return appendCallWithExceptionCheckSetResult(operation, result);
1021     }
1022     JITCompiler::Call callOperation(C_JITOperation_EStZ operation, GPRReg result, Structure* structure, GPRReg length)
1023     {
1024         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), length);
1025         return appendCallWithExceptionCheckSetResult(operation, result);
1026     }
1027     JITCompiler::Call callOperation(C_JITOperation_EStZZ operation, GPRReg result, Structure* structure, GPRReg length, unsigned minCapacity)
1028     {
1029         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), length, TrustedImm32(minCapacity));
1030         return appendCallWithExceptionCheckSetResult(operation, result);
1031     }
1032     JITCompiler::Call callOperation(C_JITOperation_EJssSt operation, GPRReg result, GPRReg arg1, Structure* structure)
1033     {
1034         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(structure));
1035         return appendCallWithExceptionCheckSetResult(operation, result);
1036     }
1037     JITCompiler::Call callOperation(C_JITOperation_EJssJss operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1038     {
1039         m_jit.setupArgumentsWithExecState(arg1, arg2);
1040         return appendCallWithExceptionCheckSetResult(operation, result);
1041     }
1042     JITCompiler::Call callOperation(C_JITOperation_EJssJssJss operation, GPRReg result, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1043     {
1044         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1045         return appendCallWithExceptionCheckSetResult(operation, result);
1046     }
1047
1048     JITCompiler::Call callOperation(S_JITOperation_ECC operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1049     {
1050         m_jit.setupArgumentsWithExecState(arg1, arg2);
1051         return appendCallWithExceptionCheckSetResult(operation, result);
1052     }
1053
1054     JITCompiler::Call callOperation(S_JITOperation_EGC operation, GPRReg result, JSGlobalObject* globalObject, GPRReg arg2)
1055     {
1056         m_jit.setupArgumentsWithExecState(TrustedImmPtr(globalObject), arg2);
1057         return appendCallWithExceptionCheckSetResult(operation, result);
1058     }
1059
1060     JITCompiler::Call callOperation(C_JITOperation_EGC operation, GPRReg result, JSGlobalObject* globalObject, GPRReg arg2)
1061     {
1062         m_jit.setupArgumentsWithExecState(TrustedImmPtr(globalObject), arg2);
1063         return appendCallWithExceptionCheckSetResult(operation, result);
1064     }
1065
1066     JITCompiler::Call callOperation(Jss_JITOperation_EZ operation, GPRReg result, GPRReg arg1)
1067     {
1068         m_jit.setupArgumentsWithExecState(arg1);
1069         return appendCallWithExceptionCheckSetResult(operation, result);
1070     }
1071
1072     JITCompiler::Call callOperation(V_JITOperation_EC operation, GPRReg arg1)
1073     {
1074         m_jit.setupArgumentsWithExecState(arg1);
1075         return appendCallWithExceptionCheck(operation);
1076     }
1077
1078     JITCompiler::Call callOperation(V_JITOperation_EC operation, JSCell* arg1)
1079     {
1080         m_jit.setupArgumentsWithExecState(TrustedImmPtr(arg1));
1081         return appendCallWithExceptionCheck(operation);
1082     }
1083
1084     JITCompiler::Call callOperation(V_JITOperation_ECIcf operation, GPRReg arg1, InlineCallFrame* inlineCallFrame)
1085     {
1086         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(inlineCallFrame));
1087         return appendCallWithExceptionCheck(operation);
1088     }
1089     JITCompiler::Call callOperation(V_JITOperation_ECCIcf operation, GPRReg arg1, GPRReg arg2, InlineCallFrame* inlineCallFrame)
1090     {
1091         m_jit.setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(inlineCallFrame));
1092         return appendCallWithExceptionCheck(operation);
1093     }
1094
1095     JITCompiler::Call callOperation(V_JITOperation_ECZ operation, GPRReg arg1, int arg2)
1096     {
1097         m_jit.setupArgumentsWithExecState(arg1, TrustedImm32(arg2));
1098         return appendCallWithExceptionCheck(operation);
1099     }
1100     JITCompiler::Call callOperation(V_JITOperation_ECC operation, GPRReg arg1, GPRReg arg2)
1101     {
1102         m_jit.setupArgumentsWithExecState(arg1, arg2);
1103         return appendCallWithExceptionCheck(operation);
1104     }
1105     JITCompiler::Call callOperation(V_JITOperation_ECC operation, GPRReg arg1, JSCell* arg2)
1106     {
1107         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(arg2));
1108         return appendCallWithExceptionCheck(operation);
1109     }
1110     JITCompiler::Call callOperation(V_JITOperation_ECC operation, JSCell* arg1, GPRReg arg2)
1111     {
1112         m_jit.setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2);
1113         return appendCallWithExceptionCheck(operation);
1114     }
1115
1116     JITCompiler::Call callOperationWithCallFrameRollbackOnException(V_JITOperation_ECb operation, void* pointer)
1117     {
1118         m_jit.setupArgumentsWithExecState(TrustedImmPtr(pointer));
1119         return appendCallWithCallFrameRollbackOnException(operation);
1120     }
1121
1122     JITCompiler::Call callOperationWithCallFrameRollbackOnException(Z_JITOperation_E operation, GPRReg result)
1123     {
1124         m_jit.setupArgumentsExecState();
1125         return appendCallWithCallFrameRollbackOnExceptionSetResult(operation, result);
1126     }
1127     JITCompiler::Call callOperation(Z_JITOperation_EC operation, GPRReg result, GPRReg arg1)
1128     {
1129         m_jit.setupArgumentsWithExecState(arg1);
1130         return appendCallWithExceptionCheckSetResult(operation, result);
1131     }
1132
1133     template<typename FunctionType>
1134     JITCompiler::Call callOperation(FunctionType operation, NoResultTag)
1135     {
1136         return callOperation(operation);
1137     }
1138     template<typename FunctionType, typename ArgumentType1>
1139     JITCompiler::Call callOperation(FunctionType operation, NoResultTag, ArgumentType1 arg1)
1140     {
1141         return callOperation(operation, arg1);
1142     }
1143     template<typename FunctionType, typename ArgumentType1, typename ArgumentType2>
1144     JITCompiler::Call callOperation(FunctionType operation, NoResultTag, ArgumentType1 arg1, ArgumentType2 arg2)
1145     {
1146         return callOperation(operation, arg1, arg2);
1147     }
1148     template<typename FunctionType, typename ArgumentType1, typename ArgumentType2, typename ArgumentType3>
1149     JITCompiler::Call callOperation(FunctionType operation, NoResultTag, ArgumentType1 arg1, ArgumentType2 arg2, ArgumentType3 arg3)
1150     {
1151         return callOperation(operation, arg1, arg2, arg3);
1152     }
1153     template<typename FunctionType, typename ArgumentType1, typename ArgumentType2, typename ArgumentType3, typename ArgumentType4>
1154     JITCompiler::Call callOperation(FunctionType operation, NoResultTag, ArgumentType1 arg1, ArgumentType2 arg2, ArgumentType3 arg3, ArgumentType4 arg4)
1155     {
1156         return callOperation(operation, arg1, arg2, arg3, arg4);
1157     }
1158     template<typename FunctionType, typename ArgumentType1, typename ArgumentType2, typename ArgumentType3, typename ArgumentType4, typename ArgumentType5>
1159     JITCompiler::Call callOperation(FunctionType operation, NoResultTag, ArgumentType1 arg1, ArgumentType2 arg2, ArgumentType3 arg3, ArgumentType4 arg4, ArgumentType5 arg5)
1160     {
1161         return callOperation(operation, arg1, arg2, arg3, arg4, arg5);
1162     }
1163
1164     JITCompiler::Call callOperation(D_JITOperation_ZZ operation, FPRReg result, GPRReg arg1, GPRReg arg2)
1165     {
1166         m_jit.setupArguments(arg1, arg2);
1167         return appendCallSetResult(operation, result);
1168     }
1169     JITCompiler::Call callOperation(D_JITOperation_D operation, FPRReg result, FPRReg arg1)
1170     {
1171         m_jit.setupArguments(arg1);
1172         return appendCallSetResult(operation, result);
1173     }
1174     JITCompiler::Call callOperation(D_JITOperation_DD operation, FPRReg result, FPRReg arg1, FPRReg arg2)
1175     {
1176         m_jit.setupArguments(arg1, arg2);
1177         return appendCallSetResult(operation, result);
1178     }
1179     JITCompiler::Call callOperation(T_JITOperation_EJss operation, GPRReg result, GPRReg arg1)
1180     {
1181         m_jit.setupArgumentsWithExecState(arg1);
1182         return appendCallWithExceptionCheckSetResult(operation, result);
1183     }
1184     JITCompiler::Call callOperation(C_JITOperation_EJscZ operation, GPRReg result, GPRReg arg1, int32_t arg2)
1185     {
1186         m_jit.setupArgumentsWithExecState(arg1, TrustedImm32(arg2));
1187         return appendCallWithExceptionCheckSetResult(operation, result);
1188     }
1189     JITCompiler::Call callOperation(C_JITOperation_EZ operation, GPRReg result, GPRReg arg1)
1190     {
1191         m_jit.setupArgumentsWithExecState(arg1);
1192         return appendCallWithExceptionCheckSetResult(operation, result);
1193     }
1194     JITCompiler::Call callOperation(C_JITOperation_EZ operation, GPRReg result, int32_t arg1)
1195     {
1196         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1));
1197         return appendCallWithExceptionCheckSetResult(operation, result);
1198     }
1199
1200     JITCompiler::Call callOperation(J_JITOperation_EJscC operation, GPRReg result, GPRReg arg1, JSCell* cell)
1201     {
1202         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(cell));
1203         return appendCallWithExceptionCheckSetResult(operation, result);
1204     }
1205
1206     JITCompiler::Call callOperation(V_JITOperation_EWs operation, WatchpointSet* watchpointSet)
1207     {
1208         m_jit.setupArgumentsWithExecState(TrustedImmPtr(watchpointSet));
1209         return appendCall(operation);
1210     }
1211
1212 #if USE(JSVALUE64)
1213     JITCompiler::Call callOperation(J_JITOperation_E operation, GPRReg result)
1214     {
1215         m_jit.setupArgumentsExecState();
1216         return appendCallWithExceptionCheckSetResult(operation, result);
1217     }
1218     JITCompiler::Call callOperation(J_JITOperation_EP operation, GPRReg result, void* pointer)
1219     {
1220         m_jit.setupArgumentsWithExecState(TrustedImmPtr(pointer));
1221         return appendCallWithExceptionCheckSetResult(operation, result);
1222     }
1223     JITCompiler::Call callOperation(Z_JITOperation_D operation, GPRReg result, FPRReg arg1)
1224     {
1225         m_jit.setupArguments(arg1);
1226         JITCompiler::Call call = m_jit.appendCall(operation);
1227         m_jit.zeroExtend32ToPtr(GPRInfo::returnValueGPR, result);
1228         return call;
1229     }
1230     JITCompiler::Call callOperation(Q_JITOperation_J operation, GPRReg result, GPRReg value)
1231     {
1232         m_jit.setupArguments(value);
1233         return appendCallSetResult(operation, result);
1234     }
1235     JITCompiler::Call callOperation(Q_JITOperation_D operation, GPRReg result, FPRReg value)
1236     {
1237         m_jit.setupArguments(value);
1238         return appendCallSetResult(operation, result);
1239     }
1240     JITCompiler::Call callOperation(J_JITOperation_EI operation, GPRReg result, UniquedStringImpl* uid)
1241     {
1242         m_jit.setupArgumentsWithExecState(TrustedImmPtr(uid));
1243         return appendCallWithExceptionCheckSetResult(operation, result);
1244     }
1245     JITCompiler::Call callOperation(J_JITOperation_EA operation, GPRReg result, GPRReg arg1)
1246     {
1247         m_jit.setupArgumentsWithExecState(arg1);
1248         return appendCallWithExceptionCheckSetResult(operation, result);
1249     }
1250     JITCompiler::Call callOperation(J_JITOperation_EAZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1251     {
1252         m_jit.setupArgumentsWithExecState(arg1, arg2);
1253         return appendCallWithExceptionCheckSetResult(operation, result);
1254     }
1255     JITCompiler::Call callOperation(J_JITOperation_EJssZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1256     {
1257         m_jit.setupArgumentsWithExecState(arg1, arg2);
1258         return appendCallWithExceptionCheckSetResult(operation, result);
1259     }
1260     JITCompiler::Call callOperation(J_JITOperation_EPS operation, GPRReg result, void* pointer, size_t size)
1261     {
1262         m_jit.setupArgumentsWithExecState(TrustedImmPtr(pointer), TrustedImmPtr(size));
1263         return appendCallWithExceptionCheckSetResult(operation, result);
1264     }
1265     JITCompiler::Call callOperation(J_JITOperation_ESS operation, GPRReg result, int startConstant, int numConstants)
1266     {
1267         m_jit.setupArgumentsWithExecState(TrustedImm32(startConstant), TrustedImm32(numConstants));
1268         return appendCallWithExceptionCheckSetResult(operation, result);
1269     }
1270     JITCompiler::Call callOperation(J_JITOperation_EPP operation, GPRReg result, GPRReg arg1, void* pointer)
1271     {
1272         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(pointer));
1273         return appendCallWithExceptionCheckSetResult(operation, result);
1274     }
1275     JITCompiler::Call callOperation(J_JITOperation_EC operation, GPRReg result, JSCell* cell)
1276     {
1277         m_jit.setupArgumentsWithExecState(TrustedImmPtr(cell));
1278         return appendCallWithExceptionCheckSetResult(operation, result);
1279     }
1280     JITCompiler::Call callOperation(J_JITOperation_ECZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1281     {
1282         m_jit.setupArgumentsWithExecState(arg1, arg2);
1283         return appendCallWithExceptionCheckSetResult(operation, result);
1284     }
1285     JITCompiler::Call callOperation(J_JITOperation_ESsiCI operation, GPRReg result, StructureStubInfo* stubInfo, GPRReg arg1, const UniquedStringImpl* uid)
1286     {
1287         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, TrustedImmPtr(uid));
1288         return appendCallWithExceptionCheckSetResult(operation, result);
1289     }
1290     JITCompiler::Call callOperation(J_JITOperation_ESsiJI operation, GPRReg result, StructureStubInfo* stubInfo, GPRReg arg1, UniquedStringImpl* uid)
1291     {
1292         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, TrustedImmPtr(uid));
1293         return appendCallWithExceptionCheckSetResult(operation, result);
1294     }
1295     JITCompiler::Call callOperation(J_JITOperation_EDA operation, GPRReg result, FPRReg arg1, GPRReg arg2)
1296     {
1297         m_jit.setupArgumentsWithExecState(arg1, arg2);
1298         return appendCallWithExceptionCheckSetResult(operation, result);
1299     }
1300     JITCompiler::Call callOperation(J_JITOperation_EJC operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1301     {
1302         m_jit.setupArgumentsWithExecState(arg1, arg2);
1303         return appendCallWithExceptionCheckSetResult(operation, result);
1304     }
1305     JITCompiler::Call callOperation(J_JITOperation_EJZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1306     {
1307         m_jit.setupArgumentsWithExecState(arg1, arg2);
1308         return appendCallWithExceptionCheckSetResult(operation, result);
1309     }
1310     JITCompiler::Call callOperation(J_JITOperation_EJA operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1311     {
1312         m_jit.setupArgumentsWithExecState(arg1, arg2);
1313         return appendCallWithExceptionCheckSetResult(operation, result);
1314     }
1315     JITCompiler::Call callOperation(J_JITOperation_EP operation, GPRReg result, GPRReg arg1)
1316     {
1317         m_jit.setupArgumentsWithExecState(arg1);
1318         return appendCallWithExceptionCheckSetResult(operation, result);
1319     }
1320     JITCompiler::Call callOperation(J_JITOperation_EZ operation, GPRReg result, GPRReg arg1)
1321     {
1322         m_jit.setupArgumentsWithExecState(arg1);
1323         return appendCallWithExceptionCheckSetResult(operation, result);
1324     }
1325     JITCompiler::Call callOperation(J_JITOperation_EZ operation, GPRReg result, int32_t arg1)
1326     {
1327         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1));
1328         return appendCallWithExceptionCheckSetResult(operation, result);
1329     }
1330     JITCompiler::Call callOperation(J_JITOperation_EZZ operation, GPRReg result, int32_t arg1, GPRReg arg2)
1331     {
1332         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1), arg2);
1333         return appendCallWithExceptionCheckSetResult(operation, result);
1334     }
1335     JITCompiler::Call callOperation(J_JITOperation_EZIcfZ operation, GPRReg result, int32_t arg1, InlineCallFrame* inlineCallFrame, GPRReg arg2)
1336     {
1337         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1), TrustedImmPtr(inlineCallFrame), arg2);
1338         return appendCallWithExceptionCheckSetResult(operation, result);
1339     }
1340
1341     JITCompiler::Call callOperation(P_JITOperation_EJS operation, GPRReg result, GPRReg value, size_t index)
1342     {
1343         m_jit.setupArgumentsWithExecState(value, TrustedImmPtr(index));
1344         return appendCallSetResult(operation, result);
1345     }
1346
1347     JITCompiler::Call callOperation(P_JITOperation_EStJ operation, GPRReg result, Structure* structure, GPRReg arg2)
1348     {
1349         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), arg2);
1350         return appendCallWithExceptionCheckSetResult(operation, result);
1351     }
1352
1353     JITCompiler::Call callOperation(C_JITOperation_EJ operation, GPRReg result, GPRReg arg1)
1354     {
1355         m_jit.setupArgumentsWithExecState(arg1);
1356         return appendCallWithExceptionCheckSetResult(operation, result);
1357     }
1358     JITCompiler::Call callOperation(C_JITOperation_EJJC operation, GPRReg result, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1359     {
1360         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1361         return appendCallWithExceptionCheckSetResult(operation, result);
1362     }
1363     JITCompiler::Call callOperation(C_JITOperation_EJZ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1364     {
1365         m_jit.setupArgumentsWithExecState(arg1, arg2);
1366         return appendCallWithExceptionCheckSetResult(operation, result);
1367     }
1368     JITCompiler::Call callOperation(C_JITOperation_EJZC operation, GPRReg result, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1369     {
1370         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1371         return appendCallWithExceptionCheckSetResult(operation, result);
1372     }
1373     JITCompiler::Call callOperation(S_JITOperation_J operation, GPRReg result, GPRReg arg1)
1374     {
1375         m_jit.setupArguments(arg1);
1376         return appendCallSetResult(operation, result);
1377     }
1378     JITCompiler::Call callOperation(S_JITOperation_EJ operation, GPRReg result, GPRReg arg1)
1379     {
1380         m_jit.setupArgumentsWithExecState(arg1);
1381         return appendCallWithExceptionCheckSetResult(operation, result);
1382     }
1383     JITCompiler::Call callOperation(J_JITOperation_EJ operation, GPRReg result, GPRReg arg1)
1384     {
1385         m_jit.setupArgumentsWithExecState(arg1);
1386         return appendCallWithExceptionCheckSetResult(operation, result);
1387     }
1388     JITCompiler::Call callOperation(S_JITOperation_EJJ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1389     {
1390         m_jit.setupArgumentsWithExecState(arg1, arg2);
1391         return appendCallWithExceptionCheckSetResult(operation, result);
1392     }
1393
1394     JITCompiler::Call callOperation(J_JITOperation_EPP operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1395     {
1396         m_jit.setupArgumentsWithExecState(arg1, arg2);
1397         return appendCallWithExceptionCheckSetResult(operation, result);
1398     }
1399     JITCompiler::Call callOperation(J_JITOperation_EJJ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1400     {
1401         m_jit.setupArgumentsWithExecState(arg1, arg2);
1402         return appendCallWithExceptionCheckSetResult(operation, result);
1403     }
1404     JITCompiler::Call callOperation(J_JITOperation_EJJ operation, GPRReg result, GPRReg arg1, MacroAssembler::TrustedImm32 imm)
1405     {
1406         m_jit.setupArgumentsWithExecState(arg1, MacroAssembler::TrustedImm64(JSValue::encode(jsNumber(imm.m_value))));
1407         return appendCallWithExceptionCheckSetResult(operation, result);
1408     }
1409     JITCompiler::Call callOperation(J_JITOperation_EJJ operation, GPRReg result, MacroAssembler::TrustedImm32 imm, GPRReg arg2)
1410     {
1411         m_jit.setupArgumentsWithExecState(MacroAssembler::TrustedImm64(JSValue::encode(jsNumber(imm.m_value))), arg2);
1412         return appendCallWithExceptionCheckSetResult(operation, result);
1413     }
1414     JITCompiler::Call callOperation(J_JITOperation_ECC operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1415     {
1416         m_jit.setupArgumentsWithExecState(arg1, arg2);
1417         return appendCallWithExceptionCheckSetResult(operation, result);
1418     }
1419     JITCompiler::Call callOperation(J_JITOperation_ECJ operation, GPRReg result, GPRReg arg1, GPRReg arg2)
1420     {
1421         m_jit.setupArgumentsWithExecState(arg1, arg2);
1422         return appendCallWithExceptionCheckSetResult(operation, result);
1423     }
1424     JITCompiler::Call callOperation(J_JITOperation_ECJ operation, GPRReg result, GPRReg arg1, JSValueRegs arg2)
1425     {
1426         m_jit.setupArgumentsWithExecState(arg1, arg2.gpr());
1427         return appendCallWithExceptionCheckSetResult(operation, result);
1428     }
1429
1430     JITCompiler::Call callOperation(V_JITOperation_EOZD operation, GPRReg arg1, GPRReg arg2, FPRReg arg3)
1431     {
1432         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1433         return appendCallWithExceptionCheck(operation);
1434     }
1435     JITCompiler::Call callOperation(V_JITOperation_EJ operation, GPRReg arg1)
1436     {
1437         m_jit.setupArgumentsWithExecState(arg1);
1438         return appendCallWithExceptionCheck(operation);
1439     }
1440     JITCompiler::Call callOperation(V_JITOperation_EJPP operation, GPRReg arg1, GPRReg arg2, void* pointer)
1441     {
1442         m_jit.setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(pointer));
1443         return appendCallWithExceptionCheck(operation);
1444     }
1445     JITCompiler::Call callOperation(V_JITOperation_ESsiJJI operation, StructureStubInfo* stubInfo, GPRReg arg1, GPRReg arg2, UniquedStringImpl* uid)
1446     {
1447         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, arg2, TrustedImmPtr(uid));
1448         return appendCallWithExceptionCheck(operation);
1449     }
1450     JITCompiler::Call callOperation(V_JITOperation_EJJJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1451     {
1452         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1453         return appendCallWithExceptionCheck(operation);
1454     }
1455     JITCompiler::Call callOperation(V_JITOperation_EPZJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1456     {
1457         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1458         return appendCallWithExceptionCheck(operation);
1459     }
1460
1461     JITCompiler::Call callOperation(V_JITOperation_EOZJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1462     {
1463         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1464         return appendCallWithExceptionCheck(operation);
1465     }
1466     JITCompiler::Call callOperation(V_JITOperation_ECJJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3)
1467     {
1468         m_jit.setupArgumentsWithExecState(arg1, arg2, arg3);
1469         return appendCallWithExceptionCheck(operation);
1470     }
1471
1472     JITCompiler::Call callOperation(D_JITOperation_EJ operation, FPRReg result, GPRReg arg1)
1473     {
1474         m_jit.setupArgumentsWithExecState(arg1);
1475         return appendCallWithExceptionCheckSetResult(operation, result);
1476     }
1477
1478     JITCompiler::Call callOperation(Z_JITOperation_EJZZ operation, GPRReg result, GPRReg arg1, unsigned arg2, unsigned arg3)
1479     {
1480         m_jit.setupArgumentsWithExecState(arg1, TrustedImm32(arg2), TrustedImm32(arg3));
1481         return appendCallWithExceptionCheckSetResult(operation, result);
1482     }
1483     JITCompiler::Call callOperation(F_JITOperation_EFJZZ operation, GPRReg result, GPRReg arg1, GPRReg arg2, unsigned arg3, GPRReg arg4)
1484     {
1485         m_jit.setupArgumentsWithExecState(arg1, arg2, TrustedImm32(arg3), arg4);
1486         return appendCallWithExceptionCheckSetResult(operation, result);
1487     }
1488     JITCompiler::Call callOperation(Z_JITOperation_EJZ operation, GPRReg result, GPRReg arg1, unsigned arg2)
1489     {
1490         m_jit.setupArgumentsWithExecState(arg1, TrustedImm32(arg2));
1491         return appendCallWithExceptionCheckSetResult(operation, result);
1492     }
1493     JITCompiler::Call callOperation(V_JITOperation_EZJZZZ operation, unsigned arg1, GPRReg arg2, unsigned arg3, GPRReg arg4, unsigned arg5)
1494     {
1495         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1), arg2, TrustedImm32(arg3), arg4, TrustedImm32(arg5));
1496         return appendCallWithExceptionCheck(operation);
1497     }
1498 #else // USE(JSVALUE32_64)
1499
1500 // EncodedJSValue in JSVALUE32_64 is a 64-bit integer. When being compiled in ARM EABI, it must be aligned even-numbered register (r0, r2 or [sp]).
1501 // To avoid assemblies from using wrong registers, let's occupy r1 or r3 with a dummy argument when necessary.
1502 #if (COMPILER_SUPPORTS(EABI) && CPU(ARM)) || CPU(MIPS)
1503 #define EABI_32BIT_DUMMY_ARG      TrustedImm32(0),
1504 #else
1505 #define EABI_32BIT_DUMMY_ARG
1506 #endif
1507
1508 // JSVALUE32_64 is a 64-bit integer that cannot be put half in an argument register and half on stack when using SH4 architecture.
1509 // To avoid this, let's occupy the 4th argument register (r7) with a dummy argument when necessary. This must only be done when there
1510 // is no other 32-bit value argument behind this 64-bit JSValue.
1511 #if CPU(SH4)
1512 #define SH4_32BIT_DUMMY_ARG      TrustedImm32(0),
1513 #else
1514 #define SH4_32BIT_DUMMY_ARG
1515 #endif
1516
1517     JITCompiler::Call callOperation(Z_JITOperation_D operation, GPRReg result, FPRReg arg1)
1518     {
1519         prepareForExternalCall();
1520         m_jit.setupArguments(arg1);
1521         JITCompiler::Call call = m_jit.appendCall(operation);
1522         m_jit.zeroExtend32ToPtr(GPRInfo::returnValueGPR, result);
1523         return call;
1524     }
1525     JITCompiler::Call callOperation(J_JITOperation_E operation, GPRReg resultTag, GPRReg resultPayload)
1526     {
1527         m_jit.setupArgumentsExecState();
1528         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1529     }
1530     JITCompiler::Call callOperation(J_JITOperation_EP operation, GPRReg resultTag, GPRReg resultPayload, void* pointer)
1531     {
1532         m_jit.setupArgumentsWithExecState(TrustedImmPtr(pointer));
1533         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1534     }
1535     JITCompiler::Call callOperation(J_JITOperation_EPP operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, void* pointer)
1536     {
1537         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(pointer));
1538         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1539     }
1540     JITCompiler::Call callOperation(J_JITOperation_EP operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1)
1541     {
1542         m_jit.setupArgumentsWithExecState(arg1);
1543         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1544     }
1545     JITCompiler::Call callOperation(J_JITOperation_EI operation, GPRReg resultTag, GPRReg resultPayload, UniquedStringImpl* uid)
1546     {
1547         m_jit.setupArgumentsWithExecState(TrustedImmPtr(uid));
1548         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1549     }
1550     JITCompiler::Call callOperation(J_JITOperation_EA operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1)
1551     {
1552         m_jit.setupArgumentsWithExecState(arg1);
1553         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1554     }
1555     JITCompiler::Call callOperation(J_JITOperation_EAZ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, GPRReg arg2)
1556     {
1557         m_jit.setupArgumentsWithExecState(arg1, arg2);
1558         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1559     }
1560     JITCompiler::Call callOperation(J_JITOperation_EJ operation, GPRReg resultPayload, GPRReg resultTag, GPRReg arg1)
1561     {
1562         m_jit.setupArgumentsWithExecState(arg1);
1563         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1564     }
1565     JITCompiler::Call callOperation(J_JITOperation_EJC operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2)
1566     {
1567         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, arg2);
1568         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1569     }
1570     JITCompiler::Call callOperation(J_JITOperation_EJssZ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, GPRReg arg2)
1571     {
1572         m_jit.setupArgumentsWithExecState(arg1, arg2);
1573         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1574     }
1575     JITCompiler::Call callOperation(J_JITOperation_EPS operation, GPRReg resultTag, GPRReg resultPayload, void* pointer, size_t size)
1576     {
1577         m_jit.setupArgumentsWithExecState(TrustedImmPtr(pointer), TrustedImmPtr(size));
1578         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1579     }
1580     JITCompiler::Call callOperation(J_JITOperation_ESS operation, GPRReg resultTag, GPRReg resultPayload, int startConstant, int numConstants)
1581     {
1582         m_jit.setupArgumentsWithExecState(TrustedImm32(startConstant), TrustedImm32(numConstants));
1583         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1584     }
1585     JITCompiler::Call callOperation(J_JITOperation_EJP operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload, void* pointer)
1586     {
1587         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImmPtr(pointer));
1588         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1589     }
1590     JITCompiler::Call callOperation(J_JITOperation_EJP operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2)
1591     {
1592         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, arg2);
1593         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1594     }
1595
1596     JITCompiler::Call callOperation(J_JITOperation_EC operation, GPRReg resultTag, GPRReg resultPayload, JSCell* cell)
1597     {
1598         m_jit.setupArgumentsWithExecState(TrustedImmPtr(cell));
1599         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1600     }
1601     JITCompiler::Call callOperation(J_JITOperation_ECZ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, GPRReg arg2)
1602     {
1603         m_jit.setupArgumentsWithExecState(arg1, arg2);
1604         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1605     }
1606     JITCompiler::Call callOperation(J_JITOperation_EJscC operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, JSCell* cell)
1607     {
1608         m_jit.setupArgumentsWithExecState(arg1, TrustedImmPtr(cell));
1609         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1610     }
1611     JITCompiler::Call callOperation(J_JITOperation_ESsiCI operation, GPRReg resultTag, GPRReg resultPayload, StructureStubInfo* stubInfo, GPRReg arg1, const UniquedStringImpl* uid)
1612     {
1613         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, TrustedImmPtr(uid));
1614         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1615     }
1616     JITCompiler::Call callOperation(J_JITOperation_ESsiJI operation, GPRReg resultTag, GPRReg resultPayload, StructureStubInfo* stubInfo, GPRReg arg1Tag, GPRReg arg1Payload, UniquedStringImpl* uid)
1617     {
1618         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1Payload, arg1Tag, TrustedImmPtr(uid));
1619         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1620     }
1621     JITCompiler::Call callOperation(J_JITOperation_ESsiJI operation, GPRReg resultTag, GPRReg resultPayload, StructureStubInfo* stubInfo, int32_t arg1Tag, GPRReg arg1Payload, UniquedStringImpl* uid)
1622     {
1623         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1Payload, TrustedImm32(arg1Tag), TrustedImmPtr(uid));
1624         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1625     }
1626     JITCompiler::Call callOperation(J_JITOperation_EDA operation, GPRReg resultTag, GPRReg resultPayload, FPRReg arg1, GPRReg arg2)
1627     {
1628         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1, arg2);
1629         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1630     }
1631     JITCompiler::Call callOperation(J_JITOperation_EJA operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2)
1632     {
1633         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, arg2);
1634         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1635     }
1636     JITCompiler::Call callOperation(J_JITOperation_EJA operation, GPRReg resultTag, GPRReg resultPayload, TrustedImm32 arg1Tag, GPRReg arg1Payload, GPRReg arg2)
1637     {
1638         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, arg2);
1639         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1640     }
1641     JITCompiler::Call callOperation(J_JITOperation_EJ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload)
1642     {
1643         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
1644         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1645     }
1646     JITCompiler::Call callOperation(J_JITOperation_EZ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1)
1647     {
1648         m_jit.setupArgumentsWithExecState(arg1);
1649         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1650     }
1651     JITCompiler::Call callOperation(J_JITOperation_EZ operation, GPRReg resultTag, GPRReg resultPayload, int32_t arg1)
1652     {
1653         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1));
1654         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1655     }
1656     JITCompiler::Call callOperation(J_JITOperation_EZIcfZ operation, GPRReg resultTag, GPRReg resultPayload, int32_t arg1, InlineCallFrame* inlineCallFrame, GPRReg arg2)
1657     {
1658         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1), TrustedImmPtr(inlineCallFrame), arg2);
1659         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1660     }
1661     JITCompiler::Call callOperation(J_JITOperation_EZZ operation, GPRReg resultTag, GPRReg resultPayload, int32_t arg1, GPRReg arg2)
1662     {
1663         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1), arg2);
1664         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1665     }
1666
1667     JITCompiler::Call callOperation(P_JITOperation_EJS operation, GPRReg result, JSValueRegs value, size_t index)
1668     {
1669         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG value.payloadGPR(), value.tagGPR(), TrustedImmPtr(index));
1670         return appendCallSetResult(operation, result);
1671     }
1672
1673     JITCompiler::Call callOperation(P_JITOperation_EStJ operation, GPRReg result, Structure* structure, GPRReg arg2Tag, GPRReg arg2Payload)
1674     {
1675         m_jit.setupArgumentsWithExecState(TrustedImmPtr(structure), arg2Payload, arg2Tag);
1676         return appendCallWithExceptionCheckSetResult(operation, result);
1677     }
1678
1679     JITCompiler::Call callOperation(C_JITOperation_EJ operation, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload)
1680     {
1681         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
1682         return appendCallWithExceptionCheckSetResult(operation, result);
1683     }
1684     JITCompiler::Call callOperation(S_JITOperation_J operation, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload)
1685     {
1686         m_jit.setupArguments(arg1Payload, arg1Tag);
1687         return appendCallSetResult(operation, result);
1688     }
1689     JITCompiler::Call callOperation(S_JITOperation_EJ operation, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload)
1690     {
1691         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
1692         return appendCallWithExceptionCheckSetResult(operation, result);
1693     }
1694
1695     JITCompiler::Call callOperation(S_JITOperation_EJJ operation, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload)
1696     {
1697         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
1698         return appendCallWithExceptionCheckSetResult(operation, result);
1699     }
1700     JITCompiler::Call callOperation(J_JITOperation_EJJ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload)
1701     {
1702         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
1703         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1704     }
1705     JITCompiler::Call callOperation(J_JITOperation_EJJ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1Tag, GPRReg arg1Payload, MacroAssembler::TrustedImm32 imm)
1706     {
1707         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG imm, TrustedImm32(JSValue::Int32Tag));
1708         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1709     }
1710     JITCompiler::Call callOperation(J_JITOperation_EJJ operation, GPRReg resultTag, GPRReg resultPayload, MacroAssembler::TrustedImm32 imm, GPRReg arg2Tag, GPRReg arg2Payload)
1711     {
1712         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG imm, TrustedImm32(JSValue::Int32Tag), SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
1713         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1714     }
1715
1716     JITCompiler::Call callOperation(J_JITOperation_ECJ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, GPRReg arg2Tag, GPRReg arg2Payload)
1717     {
1718         m_jit.setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag);
1719         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1720     }
1721     JITCompiler::Call callOperation(J_JITOperation_ECJ operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, GPRReg arg2Payload)
1722     {
1723         m_jit.setupArgumentsWithExecState(arg1, arg2Payload, MacroAssembler::TrustedImm32(JSValue::CellTag));
1724         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1725     }
1726     JITCompiler::Call callOperation(J_JITOperation_ECJ operation, JSValueRegs result, GPRReg arg1, JSValueRegs arg2)
1727     {
1728         m_jit.setupArgumentsWithExecState(arg1, arg2.payloadGPR(), arg2.tagGPR());
1729         return appendCallWithExceptionCheckSetResult(operation, result.payloadGPR(), result.tagGPR());
1730     }
1731     JITCompiler::Call callOperation(J_JITOperation_ECC operation, GPRReg resultTag, GPRReg resultPayload, GPRReg arg1, GPRReg arg2)
1732     {
1733         m_jit.setupArgumentsWithExecState(arg1, arg2);
1734         return appendCallWithExceptionCheckSetResult(operation, resultPayload, resultTag);
1735     }
1736
1737     JITCompiler::Call callOperation(V_JITOperation_EOZD operation, GPRReg arg1, GPRReg arg2, FPRReg arg3)
1738     {
1739         m_jit.setupArgumentsWithExecState(arg1, arg2, EABI_32BIT_DUMMY_ARG arg3);
1740         return appendCallWithExceptionCheck(operation);
1741     }
1742
1743     JITCompiler::Call callOperation(V_JITOperation_EJ operation, GPRReg arg1Tag, GPRReg arg1Payload)
1744     {
1745         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
1746         return appendCallWithExceptionCheck(operation);
1747     }
1748
1749     JITCompiler::Call callOperation(V_JITOperation_EJPP operation, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2, void* pointer)
1750     {
1751         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, arg2, TrustedImmPtr(pointer));
1752         return appendCallWithExceptionCheck(operation);
1753     }
1754     JITCompiler::Call callOperation(V_JITOperation_ESsiJJI operation, StructureStubInfo* stubInfo, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Payload, UniquedStringImpl* uid)
1755     {
1756         m_jit.setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1Payload, arg1Tag, arg2Payload, TrustedImm32(JSValue::CellTag), TrustedImmPtr(uid));
1757         return appendCallWithExceptionCheck(operation);
1758     }
1759     JITCompiler::Call callOperation(V_JITOperation_ECJJ operation, GPRReg arg1, GPRReg arg2Tag, GPRReg arg2Payload, GPRReg arg3Tag, GPRReg arg3Payload)
1760     {
1761         m_jit.setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag, arg3Payload, arg3Tag);
1762         return appendCallWithExceptionCheck(operation);
1763     }
1764
1765     JITCompiler::Call callOperation(V_JITOperation_EPZJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3Tag, GPRReg arg3Payload)
1766     {
1767         m_jit.setupArgumentsWithExecState(arg1, arg2, EABI_32BIT_DUMMY_ARG SH4_32BIT_DUMMY_ARG arg3Payload, arg3Tag);
1768         return appendCallWithExceptionCheck(operation);
1769     }
1770
1771     JITCompiler::Call callOperation(V_JITOperation_EOZJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3Tag, GPRReg arg3Payload)
1772     {
1773         m_jit.setupArgumentsWithExecState(arg1, arg2, EABI_32BIT_DUMMY_ARG SH4_32BIT_DUMMY_ARG arg3Payload, arg3Tag);
1774         return appendCallWithExceptionCheck(operation);
1775     }
1776     JITCompiler::Call callOperation(V_JITOperation_EOZJ operation, GPRReg arg1, GPRReg arg2, TrustedImm32 arg3Tag, GPRReg arg3Payload)
1777     {
1778         m_jit.setupArgumentsWithExecState(arg1, arg2, EABI_32BIT_DUMMY_ARG SH4_32BIT_DUMMY_ARG arg3Payload, arg3Tag);
1779         return appendCallWithExceptionCheck(operation);
1780     }
1781
1782     JITCompiler::Call callOperation(D_JITOperation_EJ operation, FPRReg result, GPRReg arg1Tag, GPRReg arg1Payload)
1783     {
1784         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
1785         return appendCallWithExceptionCheckSetResult(operation, result);
1786     }
1787
1788     JITCompiler::Call callOperation(Z_JITOperation_EJZZ operation, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload, unsigned arg2, unsigned arg3)
1789     {
1790         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG  arg1Payload, arg1Tag, TrustedImm32(arg2), TrustedImm32(arg3));
1791         return appendCallWithExceptionCheckSetResult(operation, result);
1792     }
1793     JITCompiler::Call callOperation(F_JITOperation_EFJZZ operation, GPRReg result, GPRReg arg1, GPRReg arg2Tag, GPRReg arg2Payload, unsigned arg3, GPRReg arg4)
1794     {
1795         m_jit.setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag, TrustedImm32(arg3), arg4);
1796         return appendCallWithExceptionCheckSetResult(operation, result);
1797     }
1798     JITCompiler::Call callOperation(Z_JITOperation_EJZ operation, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload, unsigned arg2)
1799     {
1800         m_jit.setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImm32(arg2));
1801         return appendCallWithExceptionCheckSetResult(operation, result);
1802     }
1803     JITCompiler::Call callOperation(V_JITOperation_EZJZZZ operation, unsigned arg1, GPRReg arg2Tag, GPRReg arg2Payload, unsigned arg3, GPRReg arg4, unsigned arg5)
1804     {
1805         m_jit.setupArgumentsWithExecState(TrustedImm32(arg1), arg2Payload, arg2Tag, TrustedImm32(arg3), arg4, TrustedImm32(arg5));
1806         return appendCallWithExceptionCheck(operation);
1807     }
1808 #undef EABI_32BIT_DUMMY_ARG
1809 #undef SH4_32BIT_DUMMY_ARG
1810
1811     template<typename FunctionType>
1812     JITCompiler::Call callOperation(
1813         FunctionType operation, JSValueRegs result)
1814     {
1815         return callOperation(operation, result.tagGPR(), result.payloadGPR());
1816     }
1817     template<typename FunctionType, typename ArgumentType1>
1818     JITCompiler::Call callOperation(
1819         FunctionType operation, JSValueRegs result, ArgumentType1 arg1)
1820     {
1821         return callOperation(operation, result.tagGPR(), result.payloadGPR(), arg1);
1822     }
1823     template<typename FunctionType, typename ArgumentType1, typename ArgumentType2>
1824     JITCompiler::Call callOperation(
1825         FunctionType operation, JSValueRegs result, ArgumentType1 arg1, ArgumentType2 arg2)
1826     {
1827         return callOperation(operation, result.tagGPR(), result.payloadGPR(), arg1, arg2);
1828     }
1829     template<
1830         typename FunctionType, typename ArgumentType1, typename ArgumentType2,
1831         typename ArgumentType3>
1832     JITCompiler::Call callOperation(
1833         FunctionType operation, JSValueRegs result, ArgumentType1 arg1, ArgumentType2 arg2,
1834         ArgumentType3 arg3)
1835     {
1836         return callOperation(operation, result.tagGPR(), result.payloadGPR(), arg1, arg2, arg3);
1837     }
1838     template<
1839         typename FunctionType, typename ArgumentType1, typename ArgumentType2,
1840         typename ArgumentType3, typename ArgumentType4>
1841     JITCompiler::Call callOperation(
1842         FunctionType operation, JSValueRegs result, ArgumentType1 arg1, ArgumentType2 arg2,
1843         ArgumentType3 arg3, ArgumentType4 arg4)
1844     {
1845         return callOperation(operation, result.tagGPR(), result.payloadGPR(), arg1, arg2, arg3, arg4);
1846     }
1847     template<
1848         typename FunctionType, typename ArgumentType1, typename ArgumentType2,
1849         typename ArgumentType3, typename ArgumentType4, typename ArgumentType5>
1850     JITCompiler::Call callOperation(
1851         FunctionType operation, JSValueRegs result, ArgumentType1 arg1, ArgumentType2 arg2,
1852         ArgumentType3 arg3, ArgumentType4 arg4, ArgumentType5 arg5)
1853     {
1854         return callOperation(
1855             operation, result.tagGPR(), result.payloadGPR(), arg1, arg2, arg3, arg4, arg5);
1856     }
1857 #endif // USE(JSVALUE32_64)
1858
1859 #if !defined(NDEBUG) && !CPU(ARM) && !CPU(MIPS) && !CPU(SH4)
1860     void prepareForExternalCall()
1861     {
1862         // We're about to call out to a "native" helper function. The helper
1863         // function is expected to set topCallFrame itself with the ExecState
1864         // that is passed to it.
1865         //
1866         // We explicitly trash topCallFrame here so that we'll know if some of
1867         // the helper functions are not setting topCallFrame when they should
1868         // be doing so. Note: the previous value in topcallFrame was not valid
1869         // anyway since it was not being updated by JIT'ed code by design.
1870
1871         for (unsigned i = 0; i < sizeof(void*) / 4; i++)
1872             m_jit.store32(TrustedImm32(0xbadbeef), reinterpret_cast<char*>(&m_jit.vm()->topCallFrame) + i * 4);
1873     }
1874 #else
1875     void prepareForExternalCall() { }
1876 #endif
1877
1878     // These methods add call instructions, with optional exception checks & setting results.
1879     JITCompiler::Call appendCallWithExceptionCheck(const FunctionPtr& function)
1880     {
1881         prepareForExternalCall();
1882         m_jit.emitStoreCodeOrigin(m_currentNode->origin.semantic);
1883         JITCompiler::Call call = m_jit.appendCall(function);
1884         m_jit.exceptionCheck();
1885         return call;
1886     }
1887     JITCompiler::Call appendCallWithCallFrameRollbackOnException(const FunctionPtr& function)
1888     {
1889         prepareForExternalCall();
1890         m_jit.emitStoreCodeOrigin(m_currentNode->origin.semantic);
1891         JITCompiler::Call call = m_jit.appendCall(function);
1892         m_jit.exceptionCheckWithCallFrameRollback();
1893         return call;
1894     }
1895     JITCompiler::Call appendCallWithExceptionCheckSetResult(const FunctionPtr& function, GPRReg result)
1896     {
1897         JITCompiler::Call call = appendCallWithExceptionCheck(function);
1898         if ((result != InvalidGPRReg) && (result != GPRInfo::returnValueGPR))
1899             m_jit.move(GPRInfo::returnValueGPR, result);
1900         return call;
1901     }
1902     JITCompiler::Call appendCallWithCallFrameRollbackOnExceptionSetResult(const FunctionPtr& function, GPRReg result)
1903     {
1904         JITCompiler::Call call = appendCallWithCallFrameRollbackOnException(function);
1905         if ((result != InvalidGPRReg) && (result != GPRInfo::returnValueGPR))
1906             m_jit.move(GPRInfo::returnValueGPR, result);
1907         return call;
1908     }
1909     JITCompiler::Call appendCallSetResult(const FunctionPtr& function, GPRReg result)
1910     {
1911         prepareForExternalCall();
1912         m_jit.emitStoreCodeOrigin(m_currentNode->origin.semantic);
1913         JITCompiler::Call call = m_jit.appendCall(function);
1914         if (result != InvalidGPRReg)
1915             m_jit.move(GPRInfo::returnValueGPR, result);
1916         return call;
1917     }
1918     JITCompiler::Call appendCall(const FunctionPtr& function)
1919     {
1920         prepareForExternalCall();
1921         m_jit.emitStoreCodeOrigin(m_currentNode->origin.semantic);
1922         return m_jit.appendCall(function);
1923     }
1924     JITCompiler::Call appendCallWithExceptionCheckSetResult(const FunctionPtr& function, GPRReg result1, GPRReg result2)
1925     {
1926         JITCompiler::Call call = appendCallWithExceptionCheck(function);
1927         m_jit.setupResults(result1, result2);
1928         return call;
1929     }
1930 #if CPU(X86)
1931     JITCompiler::Call appendCallWithExceptionCheckSetResult(const FunctionPtr& function, FPRReg result)
1932     {
1933         JITCompiler::Call call = appendCallWithExceptionCheck(function);
1934         if (result != InvalidFPRReg) {
1935             m_jit.assembler().fstpl(0, JITCompiler::stackPointerRegister);
1936             m_jit.loadDouble(JITCompiler::stackPointerRegister, result);
1937         }
1938         return call;
1939     }
1940     JITCompiler::Call appendCallSetResult(const FunctionPtr& function, FPRReg result)
1941     {
1942         JITCompiler::Call call = appendCall(function);
1943         if (result != InvalidFPRReg) {
1944             m_jit.assembler().fstpl(0, JITCompiler::stackPointerRegister);
1945             m_jit.loadDouble(JITCompiler::stackPointerRegister, result);
1946         }
1947         return call;
1948     }
1949 #elif CPU(ARM) && !CPU(ARM_HARDFP)
1950     JITCompiler::Call appendCallWithExceptionCheckSetResult(const FunctionPtr& function, FPRReg result)
1951     {
1952         JITCompiler::Call call = appendCallWithExceptionCheck(function);
1953         if (result != InvalidFPRReg)
1954             m_jit.assembler().vmov(result, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
1955         return call;
1956     }
1957     JITCompiler::Call appendCallSetResult(const FunctionPtr& function, FPRReg result)
1958     {
1959         JITCompiler::Call call = appendCall(function);
1960         if (result != InvalidFPRReg)
1961             m_jit.assembler().vmov(result, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
1962         return call;
1963     }
1964 #else // CPU(X86_64) || (CPU(ARM) && CPU(ARM_HARDFP)) || CPU(ARM64) || CPU(MIPS) || CPU(SH4)
1965     JITCompiler::Call appendCallWithExceptionCheckSetResult(const FunctionPtr& function, FPRReg result)
1966     {
1967         JITCompiler::Call call = appendCallWithExceptionCheck(function);
1968         if (result != InvalidFPRReg)
1969             m_jit.moveDouble(FPRInfo::returnValueFPR, result);
1970         return call;
1971     }
1972     JITCompiler::Call appendCallSetResult(const FunctionPtr& function, FPRReg result)
1973     {
1974         JITCompiler::Call call = appendCall(function);
1975         if (result != InvalidFPRReg)
1976             m_jit.moveDouble(FPRInfo::returnValueFPR, result);
1977         return call;
1978     }
1979 #endif
1980
1981     void branchDouble(JITCompiler::DoubleCondition cond, FPRReg left, FPRReg right, BasicBlock* destination)
1982     {
1983         return addBranch(m_jit.branchDouble(cond, left, right), destination);
1984     }
1985
1986     void branchDoubleNonZero(FPRReg value, FPRReg scratch, BasicBlock* destination)
1987     {
1988         return addBranch(m_jit.branchDoubleNonZero(value, scratch), destination);
1989     }
1990
1991     template<typename T, typename U>
1992     void branch32(JITCompiler::RelationalCondition cond, T left, U right, BasicBlock* destination)
1993     {
1994         return addBranch(m_jit.branch32(cond, left, right), destination);
1995     }
1996
1997     template<typename T, typename U>
1998     void branchTest32(JITCompiler::ResultCondition cond, T value, U mask, BasicBlock* destination)
1999     {
2000         return addBranch(m_jit.branchTest32(cond, value, mask), destination);
2001     }
2002
2003     template<typename T>
2004     void branchTest32(JITCompiler::ResultCondition cond, T value, BasicBlock* destination)
2005     {
2006         return addBranch(m_jit.branchTest32(cond, value), destination);
2007     }
2008
2009 #if USE(JSVALUE64)
2010     template<typename T, typename U>
2011     void branch64(JITCompiler::RelationalCondition cond, T left, U right, BasicBlock* destination)
2012     {
2013         return addBranch(m_jit.branch64(cond, left, right), destination);
2014     }
2015 #endif
2016
2017     template<typename T, typename U>
2018     void branch8(JITCompiler::RelationalCondition cond, T left, U right, BasicBlock* destination)
2019     {
2020         return addBranch(m_jit.branch8(cond, left, right), destination);
2021     }
2022
2023     template<typename T, typename U>
2024     void branchPtr(JITCompiler::RelationalCondition cond, T left, U right, BasicBlock* destination)
2025     {
2026         return addBranch(m_jit.branchPtr(cond, left, right), destination);
2027     }
2028
2029     template<typename T, typename U>
2030     void branchTestPtr(JITCompiler::ResultCondition cond, T value, U mask, BasicBlock* destination)
2031     {
2032         return addBranch(m_jit.branchTestPtr(cond, value, mask), destination);
2033     }
2034
2035     template<typename T>
2036     void branchTestPtr(JITCompiler::ResultCondition cond, T value, BasicBlock* destination)
2037     {
2038         return addBranch(m_jit.branchTestPtr(cond, value), destination);
2039     }
2040
2041     template<typename T, typename U>
2042     void branchTest8(JITCompiler::ResultCondition cond, T value, U mask, BasicBlock* destination)
2043     {
2044         return addBranch(m_jit.branchTest8(cond, value, mask), destination);
2045     }
2046
2047     template<typename T>
2048     void branchTest8(JITCompiler::ResultCondition cond, T value, BasicBlock* destination)
2049     {
2050         return addBranch(m_jit.branchTest8(cond, value), destination);
2051     }
2052
2053     enum FallThroughMode {
2054         AtFallThroughPoint,
2055         ForceJump
2056     };
2057     void jump(BasicBlock* destination, FallThroughMode fallThroughMode = AtFallThroughPoint)
2058     {
2059         if (destination == nextBlock()
2060             && fallThroughMode == AtFallThroughPoint)
2061             return;
2062         addBranch(m_jit.jump(), destination);
2063     }
2064
2065     void addBranch(const MacroAssembler::Jump& jump, BasicBlock* destination)
2066     {
2067         m_branches.append(BranchRecord(jump, destination));
2068     }
2069     void addBranch(const MacroAssembler::JumpList& jump, BasicBlock* destination);
2070
2071     void linkBranches();
2072
2073     void dump(const char* label = 0);
2074
2075     bool betterUseStrictInt52(Node* node)
2076     {
2077         return !generationInfo(node).isInt52();
2078     }
2079     bool betterUseStrictInt52(Edge edge)
2080     {
2081         return betterUseStrictInt52(edge.node());
2082     }
2083
2084     bool compare(Node*, MacroAssembler::RelationalCondition, MacroAssembler::DoubleCondition, S_JITOperation_EJJ);
2085     bool compilePeepHoleBranch(Node*, MacroAssembler::RelationalCondition, MacroAssembler::DoubleCondition, S_JITOperation_EJJ);
2086     void compilePeepHoleInt32Branch(Node*, Node* branchNode, JITCompiler::RelationalCondition);
2087     void compilePeepHoleInt52Branch(Node*, Node* branchNode, JITCompiler::RelationalCondition);
2088     void compilePeepHoleBooleanBranch(Node*, Node* branchNode, JITCompiler::RelationalCondition);
2089     void compilePeepHoleDoubleBranch(Node*, Node* branchNode, JITCompiler::DoubleCondition);
2090     void compilePeepHoleObjectEquality(Node*, Node* branchNode);
2091     void compilePeepHoleObjectStrictEquality(Edge objectChild, Edge otherChild, Node* branchNode);
2092     void compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode);
2093     void compileObjectEquality(Node*);
2094     void compileObjectStrictEquality(Edge objectChild, Edge otherChild);
2095     void compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild);
2096     void compileObjectOrOtherLogicalNot(Edge value);
2097     void compileLogicalNot(Node*);
2098     void compileStringEquality(
2099         Node*, GPRReg leftGPR, GPRReg rightGPR, GPRReg lengthGPR,
2100         GPRReg leftTempGPR, GPRReg rightTempGPR, GPRReg leftTemp2GPR,
2101         GPRReg rightTemp2GPR, JITCompiler::JumpList fastTrue,
2102         JITCompiler::JumpList fastSlow);
2103     void compileStringEquality(Node*);
2104     void compileStringIdentEquality(Node*);
2105     void compileStringToUntypedEquality(Node*, Edge stringEdge, Edge untypedEdge);
2106     void compileStringIdentToNotStringVarEquality(Node*, Edge stringEdge, Edge notStringVarEdge);
2107     void compileStringZeroLength(Node*);
2108     void compileMiscStrictEq(Node*);
2109
2110     void emitObjectOrOtherBranch(Edge value, BasicBlock* taken, BasicBlock* notTaken);
2111     void emitStringBranch(Edge value, BasicBlock* taken, BasicBlock* notTaken);
2112     void emitBranch(Node*);
2113
2114     struct StringSwitchCase {
2115         StringSwitchCase() { }
2116
2117         StringSwitchCase(StringImpl* string, BasicBlock* target)
2118             : string(string)
2119             , target(target)
2120         {
2121         }
2122
2123         bool operator<(const StringSwitchCase& other) const
2124         {
2125             return stringLessThan(*string, *other.string);
2126         }
2127
2128         StringImpl* string;
2129         BasicBlock* target;
2130     };
2131
2132     void emitSwitchIntJump(SwitchData*, GPRReg value, GPRReg scratch);
2133     void emitSwitchImm(Node*, SwitchData*);
2134     void emitSwitchCharStringJump(SwitchData*, GPRReg value, GPRReg scratch);
2135     void emitSwitchChar(Node*, SwitchData*);
2136     void emitBinarySwitchStringRecurse(
2137         SwitchData*, const Vector<StringSwitchCase>&, unsigned numChecked,
2138         unsigned begin, unsigned end, GPRReg buffer, GPRReg length, GPRReg temp,
2139         unsigned alreadyCheckedLength, bool checkedExactLength);
2140     void emitSwitchStringOnString(SwitchData*, GPRReg string);
2141     void emitSwitchString(Node*, SwitchData*);
2142     void emitSwitch(Node*);
2143
2144     void compileToStringOrCallStringConstructorOnCell(Node*);
2145     void compileNewStringObject(Node*);
2146
2147     void compileNewTypedArray(Node*);
2148
2149     void compileInt32Compare(Node*, MacroAssembler::RelationalCondition);
2150     void compileInt52Compare(Node*, MacroAssembler::RelationalCondition);
2151     void compileBooleanCompare(Node*, MacroAssembler::RelationalCondition);
2152     void compileDoubleCompare(Node*, MacroAssembler::DoubleCondition);
2153
2154     bool compileStrictEq(Node*);
2155
2156     void compileAllocatePropertyStorage(Node*);
2157     void compileReallocatePropertyStorage(Node*);
2158
2159 #if USE(JSVALUE32_64)
2160     template<typename BaseOperandType, typename PropertyOperandType, typename ValueOperandType, typename TagType>
2161     void compileContiguousPutByVal(Node*, BaseOperandType&, PropertyOperandType&, ValueOperandType&, GPRReg valuePayloadReg, TagType valueTag);
2162 #endif
2163     void compileDoublePutByVal(Node*, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property);
2164     bool putByValWillNeedExtraRegister(ArrayMode arrayMode)
2165     {
2166         return arrayMode.mayStoreToHole();
2167     }
2168     GPRReg temporaryRegisterForPutByVal(GPRTemporary&, ArrayMode);
2169     GPRReg temporaryRegisterForPutByVal(GPRTemporary& temporary, Node* node)
2170     {
2171         return temporaryRegisterForPutByVal(temporary, node->arrayMode());
2172     }
2173
2174     void compileGetCharCodeAt(Node*);
2175     void compileGetByValOnString(Node*);
2176     void compileFromCharCode(Node*);
2177
2178     void compileGetByValOnDirectArguments(Node*);
2179     void compileGetByValOnScopedArguments(Node*);
2180
2181     void compileGetScope(Node*);
2182     void compileSkipScope(Node*);
2183
2184     void compileGetArrayLength(Node*);
2185
2186     void compileValueRep(Node*);
2187     void compileDoubleRep(Node*);
2188
2189     void compileValueToInt32(Node*);
2190     void compileUInt32ToNumber(Node*);
2191     void compileDoubleAsInt32(Node*);
2192     void compileAdd(Node*);
2193     void compileMakeRope(Node*);
2194     void compileArithClz32(Node*);
2195     void compileArithSub(Node*);
2196     void compileArithNegate(Node*);
2197     void compileArithMul(Node*);
2198     void compileArithDiv(Node*);
2199     void compileArithMod(Node*);
2200     void compileArithPow(Node*);
2201     void compileArithRound(Node*);
2202     void compileArithSqrt(Node*);
2203     void compileArithLog(Node*);
2204     void compileConstantStoragePointer(Node*);
2205     void compileGetIndexedPropertyStorage(Node*);
2206     JITCompiler::Jump jumpForTypedArrayOutOfBounds(Node*, GPRReg baseGPR, GPRReg indexGPR);
2207     void emitTypedArrayBoundsCheck(Node*, GPRReg baseGPR, GPRReg indexGPR);
2208     void compileGetTypedArrayByteOffset(Node*);
2209     void compileGetByValOnIntTypedArray(Node*, TypedArrayType);
2210     void compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node*, TypedArrayType);
2211     void compileGetByValOnFloatTypedArray(Node*, TypedArrayType);
2212     void compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node*, TypedArrayType);
2213     void compileNewFunction(Node*);
2214     void compileForwardVarargs(Node*);
2215     void compileCreateActivation(Node*);
2216     void compileCreateDirectArguments(Node*);
2217     void compileGetFromArguments(Node*);
2218     void compilePutToArguments(Node*);
2219     void compileCreateScopedArguments(Node*);
2220     void compileCreateClonedArguments(Node*);
2221     void compileNotifyWrite(Node*);
2222     bool compileRegExpExec(Node*);
2223     void compileIsObjectOrNull(Node*);
2224     void compileIsFunction(Node*);
2225     void compileTypeOf(Node*);
2226
2227     void moveTrueTo(GPRReg);
2228     void moveFalseTo(GPRReg);
2229     void blessBoolean(GPRReg);
2230
2231     // size can be an immediate or a register, and must be in bytes. If size is a register,
2232     // it must be a different register than resultGPR. Emits code that place a pointer to
2233     // the end of the allocation. The returned jump is the jump to the slow path.
2234     template<typename SizeType>
2235     MacroAssembler::Jump emitAllocateBasicStorage(SizeType size, GPRReg resultGPR)
2236     {
2237         CopiedAllocator* copiedAllocator = &m_jit.vm()->heap.storageAllocator();
2238
2239         // It's invalid to allocate zero bytes in CopiedSpace.
2240 #ifndef NDEBUG
2241         m_jit.move(size, resultGPR);
2242         MacroAssembler::Jump nonZeroSize = m_jit.branchTest32(MacroAssembler::NonZero, resultGPR);
2243         m_jit.abortWithReason(DFGBasicStorageAllocatorZeroSize);
2244         nonZeroSize.link(&m_jit);
2245 #endif
2246
2247         m_jit.loadPtr(&copiedAllocator->m_currentRemaining, resultGPR);
2248         MacroAssembler::Jump slowPath = m_jit.branchSubPtr(JITCompiler::Signed, size, resultGPR);
2249         m_jit.storePtr(resultGPR, &copiedAllocator->m_currentRemaining);
2250         m_jit.negPtr(resultGPR);
2251         m_jit.addPtr(JITCompiler::AbsoluteAddress(&copiedAllocator->m_currentPayloadEnd), resultGPR);
2252
2253         return slowPath;
2254     }
2255
2256     // Allocator for a cell of a specific size.
2257     template <typename StructureType> // StructureType can be GPR or ImmPtr.
2258     void emitAllocateJSCell(GPRReg resultGPR, GPRReg allocatorGPR, StructureType structure,
2259         GPRReg scratchGPR, MacroAssembler::JumpList& slowPath)
2260     {
2261         m_jit.loadPtr(MacroAssembler::Address(allocatorGPR, MarkedAllocator::offsetOfFreeListHead()), resultGPR);
2262         slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, resultGPR));
2263
2264         // The object is half-allocated: we have what we know is a fresh object, but
2265         // it's still on the GC's free list.
2266         m_jit.loadPtr(MacroAssembler::Address(resultGPR), scratchGPR);
2267         m_jit.storePtr(scratchGPR, MacroAssembler::Address(allocatorGPR, MarkedAllocator::offsetOfFreeListHead()));
2268
2269         // Initialize the object's Structure.
2270         m_jit.emitStoreStructureWithTypeInfo(structure, resultGPR, scratchGPR);
2271     }
2272
2273     // Allocator for an object of a specific size.
2274     template <typename StructureType, typename StorageType> // StructureType and StorageType can be GPR or ImmPtr.
2275     void emitAllocateJSObject(GPRReg resultGPR, GPRReg allocatorGPR, StructureType structure,
2276         StorageType storage, GPRReg scratchGPR, MacroAssembler::JumpList& slowPath)
2277     {
2278         emitAllocateJSCell(resultGPR, allocatorGPR, structure, scratchGPR, slowPath);
2279
2280         // Initialize the object's property storage pointer.
2281         m_jit.storePtr(storage, MacroAssembler::Address(resultGPR, JSObject::butterflyOffset()));
2282     }
2283
2284     template <typename ClassType, typename StructureType, typename StorageType> // StructureType and StorageType can be GPR or ImmPtr.
2285     void emitAllocateJSObjectWithKnownSize(
2286         GPRReg resultGPR, StructureType structure, StorageType storage, GPRReg scratchGPR1,
2287         GPRReg scratchGPR2, MacroAssembler::JumpList& slowPath, size_t size)
2288     {
2289         MarkedAllocator* allocator = &m_jit.vm()->heap.allocatorForObjectOfType<ClassType>(size);
2290         m_jit.move(TrustedImmPtr(allocator), scratchGPR1);
2291         emitAllocateJSObject(resultGPR, scratchGPR1, structure, storage, scratchGPR2, slowPath);
2292     }
2293
2294     // Convenience allocator for a built-in object.
2295     template <typename ClassType, typename StructureType, typename StorageType> // StructureType and StorageType can be GPR or ImmPtr.
2296     void emitAllocateJSObject(GPRReg resultGPR, StructureType structure, StorageType storage,
2297         GPRReg scratchGPR1, GPRReg scratchGPR2, MacroAssembler::JumpList& slowPath)
2298     {
2299         emitAllocateJSObjectWithKnownSize<ClassType>(
2300             resultGPR, structure, storage, scratchGPR1, scratchGPR2, slowPath,
2301             ClassType::allocationSize(0));
2302     }
2303
2304     template <typename ClassType, typename StructureType> // StructureType and StorageType can be GPR or ImmPtr.
2305     void emitAllocateVariableSizedJSObject(GPRReg resultGPR, StructureType structure, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, MacroAssembler::JumpList& slowPath)
2306     {
2307         static_assert(!(MarkedSpace::preciseStep & (MarkedSpace::preciseStep - 1)), "MarkedSpace::preciseStep must be a power of two.");
2308         static_assert(!(MarkedSpace::impreciseStep & (MarkedSpace::impreciseStep - 1)), "MarkedSpace::impreciseStep must be a power of two.");
2309
2310         MarkedSpace::Subspace& subspace = m_jit.vm()->heap.subspaceForObjectOfType<ClassType>();
2311         m_jit.add32(TrustedImm32(MarkedSpace::preciseStep - 1), allocationSize);
2312         MacroAssembler::Jump notSmall = m_jit.branch32(MacroAssembler::AboveOrEqual, allocationSize, TrustedImm32(MarkedSpace::preciseCutoff));
2313         m_jit.rshift32(allocationSize, TrustedImm32(getLSBSet(MarkedSpace::preciseStep)), scratchGPR1);
2314         m_jit.mul32(TrustedImm32(sizeof(MarkedAllocator)), scratchGPR1, scratchGPR1);
2315         m_jit.addPtr(MacroAssembler::TrustedImmPtr(&subspace.preciseAllocators[0]), scratchGPR1);
2316
2317         MacroAssembler::Jump selectedSmallSpace = m_jit.jump();
2318         notSmall.link(&m_jit);
2319         slowPath.append(m_jit.branch32(MacroAssembler::AboveOrEqual, allocationSize, TrustedImm32(MarkedSpace::impreciseCutoff)));
2320         m_jit.rshift32(allocationSize, TrustedImm32(getLSBSet(MarkedSpace::impreciseStep)), scratchGPR1);
2321         m_jit.mul32(TrustedImm32(sizeof(MarkedAllocator)), scratchGPR1, scratchGPR1);
2322         m_jit.addPtr(MacroAssembler::TrustedImmPtr(&subspace.impreciseAllocators[0]), scratchGPR1);
2323
2324         selectedSmallSpace.link(&m_jit);
2325
2326         emitAllocateJSObject(resultGPR, scratchGPR1, structure, TrustedImmPtr(0), scratchGPR2, slowPath);
2327     }
2328
2329     template <typename T>
2330     void emitAllocateDestructibleObject(GPRReg resultGPR, Structure* structure,
2331         GPRReg scratchGPR1, GPRReg scratchGPR2, MacroAssembler::JumpList& slowPath)
2332     {
2333         emitAllocateJSObject<T>(resultGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR1, scratchGPR2, slowPath);
2334         m_jit.storePtr(TrustedImmPtr(structure->classInfo()), MacroAssembler::Address(resultGPR, JSDestructibleObject::classInfoOffset()));
2335     }
2336
2337     void emitAllocateJSArray(GPRReg resultGPR, Structure*, GPRReg storageGPR, unsigned numElements);
2338
2339     void emitGetLength(InlineCallFrame*, GPRReg lengthGPR, bool includeThis = false);
2340     void emitGetLength(CodeOrigin, GPRReg lengthGPR, bool includeThis = false);
2341     void emitGetCallee(CodeOrigin, GPRReg calleeGPR);
2342     void emitGetArgumentStart(CodeOrigin, GPRReg startGPR);
2343
2344     // Generate an OSR exit fuzz check. Returns Jump() if OSR exit fuzz is not enabled, or if
2345     // it's in training mode.
2346     MacroAssembler::Jump emitOSRExitFuzzCheck();
2347
2348     // Add a speculation check.
2349     void speculationCheck(ExitKind, JSValueSource, Node*, MacroAssembler::Jump jumpToFail);
2350     void speculationCheck(ExitKind, JSValueSource, Node*, const MacroAssembler::JumpList& jumpsToFail);
2351
2352     // Add a speculation check without additional recovery, and with a promise to supply a jump later.
2353     OSRExitJumpPlaceholder speculationCheck(ExitKind, JSValueSource, Node*);
2354     OSRExitJumpPlaceholder speculationCheck(ExitKind, JSValueSource, Edge);
2355     void speculationCheck(ExitKind, JSValueSource, Edge, MacroAssembler::Jump jumpToFail);
2356     void speculationCheck(ExitKind, JSValueSource, Edge, const MacroAssembler::JumpList& jumpsToFail);
2357     // Add a speculation check with additional recovery.
2358     void speculationCheck(ExitKind, JSValueSource, Node*, MacroAssembler::Jump jumpToFail, const SpeculationRecovery&);
2359     void speculationCheck(ExitKind, JSValueSource, Edge, MacroAssembler::Jump jumpToFail, const SpeculationRecovery&);
2360
2361     void emitInvalidationPoint(Node*);
2362
2363     // Called when we statically determine that a speculation will fail.
2364     void terminateSpeculativeExecution(ExitKind, JSValueRegs, Node*);
2365     void terminateSpeculativeExecution(ExitKind, JSValueRegs, Edge);
2366
2367     // Helpers for performing type checks on an edge stored in the given registers.
2368     bool needsTypeCheck(Edge edge, SpeculatedType typesPassedThrough) { return m_interpreter.needsTypeCheck(edge, typesPassedThrough); }
2369     void typeCheck(JSValueSource, Edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail);
2370
2371     void speculateCellTypeWithoutTypeFiltering(Edge, GPRReg cellGPR, JSType);
2372     void speculateCellType(Edge, GPRReg cellGPR, SpeculatedType, JSType);
2373
2374     void speculateInt32(Edge);
2375 #if USE(JSVALUE64)
2376     void convertMachineInt(Edge, GPRReg resultGPR);
2377     void speculateMachineInt(Edge);
2378     void speculateDoubleRepMachineInt(Edge);
2379 #endif // USE(JSVALUE64)
2380     void speculateNumber(Edge);
2381     void speculateRealNumber(Edge);
2382     void speculateDoubleRepReal(Edge);
2383     void speculateBoolean(Edge);
2384     void speculateCell(Edge);
2385     void speculateObject(Edge);
2386     void speculateFunction(Edge);
2387     void speculateFinalObject(Edge);
2388     void speculateObjectOrOther(Edge);
2389     void speculateString(Edge edge, GPRReg cell);
2390     void speculateStringIdentAndLoadStorage(Edge edge, GPRReg string, GPRReg storage);
2391     void speculateStringIdent(Edge edge, GPRReg string);
2392     void speculateStringIdent(Edge);
2393     void speculateString(Edge);
2394     void speculateNotStringVar(Edge);
2395     template<typename StructureLocationType>
2396     void speculateStringObjectForStructure(Edge, StructureLocationType);
2397     void speculateStringObject(Edge, GPRReg);
2398     void speculateStringObject(Edge);
2399     void speculateStringOrStringObject(Edge);
2400     void speculateNotCell(Edge);
2401     void speculateOther(Edge);
2402     void speculateMisc(Edge, JSValueRegs);
2403     void speculateMisc(Edge);
2404     void speculate(Node*, Edge);
2405
2406     JITCompiler::Jump jumpSlowForUnwantedArrayMode(GPRReg tempWithIndexingTypeReg, ArrayMode, IndexingType);
2407     JITCompiler::JumpList jumpSlowForUnwantedArrayMode(GPRReg tempWithIndexingTypeReg, ArrayMode);
2408     void checkArray(Node*);
2409     void arrayify(Node*, GPRReg baseReg, GPRReg propertyReg);
2410     void arrayify(Node*);
2411
2412     template<bool strict>
2413     GPRReg fillSpeculateInt32Internal(Edge, DataFormat& returnFormat);
2414
2415     // It is possible, during speculative generation, to reach a situation in which we
2416     // can statically determine a speculation will fail (for example, when two nodes
2417     // will make conflicting speculations about the same operand). In such cases this
2418     // flag is cleared, indicating no further code generation should take place.
2419     bool m_compileOkay;
2420
2421     void recordSetLocal(
2422         VirtualRegister bytecodeReg, VirtualRegister machineReg, DataFormat format)
2423     {
2424         m_stream->appendAndLog(VariableEvent::setLocal(bytecodeReg, machineReg, format));
2425     }
2426
2427     void recordSetLocal(DataFormat format)
2428     {
2429         VariableAccessData* variable = m_currentNode->variableAccessData();
2430         recordSetLocal(variable->local(), variable->machineLocal(), format);
2431     }
2432
2433     GenerationInfo& generationInfoFromVirtualRegister(VirtualRegister virtualRegister)
2434     {
2435         return m_generationInfo[virtualRegister.toLocal()];
2436     }
2437
2438     GenerationInfo& generationInfo(Node* node)
2439     {
2440         return generationInfoFromVirtualRegister(node->virtualRegister());
2441     }
2442
2443     GenerationInfo& generationInfo(Edge edge)
2444     {
2445         return generationInfo(edge.node());
2446     }
2447
2448     // The JIT, while also provides MacroAssembler functionality.
2449     JITCompiler& m_jit;
2450
2451     // The current node being generated.
2452     BasicBlock* m_block;
2453     Node* m_currentNode;
2454     NodeType m_lastGeneratedNode;
2455     bool m_canExit;
2456     unsigned m_indexInBlock;
2457     // Virtual and physical register maps.
2458     Vector<GenerationInfo, 32> m_generationInfo;
2459     RegisterBank<GPRInfo> m_gprs;
2460     RegisterBank<FPRInfo> m_fprs;
2461
2462     Vector<MacroAssembler::Label> m_osrEntryHeads;
2463
2464     struct BranchRecord {
2465         BranchRecord(MacroAssembler::Jump jump, BasicBlock* destination)
2466             : jump(jump)
2467             , destination(destination)
2468         {
2469         }
2470
2471         MacroAssembler::Jump jump;
2472         BasicBlock* destination;
2473     };
2474     Vector<BranchRecord, 8> m_branches;
2475
2476     CodeOrigin m_codeOriginForExitTarget;
2477     CodeOrigin m_codeOriginForExitProfile;
2478
2479     InPlaceAbstractState m_state;
2480     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
2481
2482     VariableEventStream* m_stream;
2483     MinifiedGraph* m_minifiedGraph;
2484
2485     bool m_isCheckingArgumentTypes;
2486
2487     Vector<std::unique_ptr<SlowPathGenerator>, 8> m_slowPathGenerators;
2488     Vector<SilentRegisterSavePlan> m_plans;
2489 };
2490
2491
2492 // === Operand types ===
2493 //
2494 // These classes are used to lock the operands to a node into machine
2495 // registers. These classes implement of pattern of locking a value
2496 // into register at the point of construction only if it is already in
2497 // registers, and otherwise loading it lazily at the point it is first
2498 // used. We do so in order to attempt to avoid spilling one operand
2499 // in order to make space available for another.
2500
2501 class JSValueOperand {
2502 public:
2503     explicit JSValueOperand(SpeculativeJIT* jit, Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2504         : m_jit(jit)
2505         , m_edge(edge)
2506 #if USE(JSVALUE64)
2507         , m_gprOrInvalid(InvalidGPRReg)
2508 #elif USE(JSVALUE32_64)
2509         , m_isDouble(false)
2510 #endif
2511     {
2512         ASSERT(m_jit);
2513         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
2514 #if USE(JSVALUE64)
2515         if (jit->isFilled(node()))
2516             gpr();
2517 #elif USE(JSVALUE32_64)
2518         m_register.pair.tagGPR = InvalidGPRReg;
2519         m_register.pair.payloadGPR = InvalidGPRReg;
2520         if (jit->isFilled(node()))
2521             fill();
2522 #endif
2523     }
2524
2525     ~JSValueOperand()
2526     {
2527 #if USE(JSVALUE64)
2528         ASSERT(m_gprOrInvalid != InvalidGPRReg);
2529         m_jit->unlock(m_gprOrInvalid);
2530 #elif USE(JSVALUE32_64)
2531         if (m_isDouble) {
2532             ASSERT(m_register.fpr != InvalidFPRReg);
2533             m_jit->unlock(m_register.fpr);
2534         } else {
2535             ASSERT(m_register.pair.tagGPR != InvalidGPRReg && m_register.pair.payloadGPR != InvalidGPRReg);
2536             m_jit->unlock(m_register.pair.tagGPR);
2537             m_jit->unlock(m_register.pair.payloadGPR);
2538         }
2539 #endif
2540     }
2541
2542     Edge edge() const
2543     {
2544         return m_edge;
2545     }
2546
2547     Node* node() const
2548     {
2549         return edge().node();
2550     }
2551
2552 #if USE(JSVALUE64)
2553     GPRReg gpr()
2554     {
2555         if (m_gprOrInvalid == InvalidGPRReg)
2556             m_gprOrInvalid = m_jit->fillJSValue(m_edge);
2557         return m_gprOrInvalid;
2558     }
2559     JSValueRegs jsValueRegs()
2560     {
2561         return JSValueRegs(gpr());
2562     }
2563 #elif USE(JSVALUE32_64)
2564     bool isDouble() { return m_isDouble; }
2565
2566     void fill()
2567     {
2568         if (m_register.pair.tagGPR == InvalidGPRReg && m_register.pair.payloadGPR == InvalidGPRReg)
2569             m_isDouble = !m_jit->fillJSValue(m_edge, m_register.pair.tagGPR, m_register.pair.payloadGPR, m_register.fpr);
2570     }
2571
2572     GPRReg tagGPR()
2573     {
2574         fill();
2575         ASSERT(!m_isDouble);
2576         return m_register.pair.tagGPR;
2577     }
2578
2579     GPRReg payloadGPR()
2580     {
2581         fill();
2582         ASSERT(!m_isDouble);
2583         return m_register.pair.payloadGPR;
2584     }
2585
2586     JSValueRegs jsValueRegs()
2587     {
2588         return JSValueRegs(tagGPR(), payloadGPR());
2589     }
2590
2591     GPRReg gpr(WhichValueWord which)
2592     {
2593         return jsValueRegs().gpr(which);
2594     }
2595
2596     FPRReg fpr()
2597     {
2598         fill();
2599         ASSERT(m_isDouble);
2600         return m_register.fpr;
2601     }
2602 #endif
2603
2604     void use()
2605     {
2606         m_jit->use(node());
2607     }
2608
2609 private:
2610     SpeculativeJIT* m_jit;
2611     Edge m_edge;
2612 #if USE(JSVALUE64)
2613     GPRReg m_gprOrInvalid;
2614 #elif USE(JSVALUE32_64)
2615     union {
2616         struct {
2617             GPRReg tagGPR;
2618             GPRReg payloadGPR;
2619         } pair;
2620         FPRReg fpr;
2621     } m_register;
2622     bool m_isDouble;
2623 #endif
2624 };
2625
2626 class StorageOperand {
2627 public:
2628     explicit StorageOperand(SpeculativeJIT* jit, Edge edge)
2629         : m_jit(jit)
2630         , m_edge(edge)
2631         , m_gprOrInvalid(InvalidGPRReg)
2632     {
2633         ASSERT(m_jit);
2634         ASSERT(edge.useKind() == UntypedUse || edge.useKind() == KnownCellUse);
2635         if (jit->isFilled(node()))
2636             gpr();
2637     }
2638
2639     ~StorageOperand()
2640     {
2641         ASSERT(m_gprOrInvalid != InvalidGPRReg);
2642         m_jit->unlock(m_gprOrInvalid);
2643     }
2644
2645     Edge edge() const
2646     {
2647         return m_edge;
2648     }
2649
2650     Node* node() const
2651     {
2652         return edge().node();
2653     }
2654
2655     GPRReg gpr()
2656     {
2657         if (m_gprOrInvalid == InvalidGPRReg)
2658             m_gprOrInvalid = m_jit->fillStorage(edge());
2659         return m_gprOrInvalid;
2660     }
2661
2662     void use()
2663     {
2664         m_jit->use(node());
2665     }
2666
2667 private:
2668     SpeculativeJIT* m_jit;
2669     Edge m_edge;
2670     GPRReg m_gprOrInvalid;
2671 };
2672
2673
2674 // === Temporaries ===
2675 //
2676 // These classes are used to allocate temporary registers.
2677 // A mechanism is provided to attempt to reuse the registers
2678 // currently allocated to child nodes whose value is consumed
2679 // by, and not live after, this operation.
2680
2681 enum ReuseTag { Reuse };
2682
2683 class GPRTemporary {
2684 public:
2685     GPRTemporary();
2686     GPRTemporary(SpeculativeJIT*);
2687     GPRTemporary(SpeculativeJIT*, GPRReg specific);
2688     template<typename T>
2689     GPRTemporary(SpeculativeJIT* jit, ReuseTag, T& operand)
2690         : m_jit(jit)
2691         , m_gpr(InvalidGPRReg)
2692     {
2693         if (m_jit->canReuse(operand.node()))
2694             m_gpr = m_jit->reuse(operand.gpr());
2695         else
2696             m_gpr = m_jit->allocate();
2697     }
2698     template<typename T1, typename T2>
2699     GPRTemporary(SpeculativeJIT* jit, ReuseTag, T1& op1, T2& op2)
2700         : m_jit(jit)
2701         , m_gpr(InvalidGPRReg)
2702     {
2703         if (m_jit->canReuse(op1.node()))
2704             m_gpr = m_jit->reuse(op1.gpr());
2705         else if (m_jit->canReuse(op2.node()))
2706             m_gpr = m_jit->reuse(op2.gpr());
2707         else
2708             m_gpr = m_jit->allocate();
2709     }
2710 #if USE(JSVALUE32_64)
2711     GPRTemporary(SpeculativeJIT*, ReuseTag, JSValueOperand&, WhichValueWord);
2712 #endif
2713
2714     void adopt(GPRTemporary&);
2715
2716     ~GPRTemporary()
2717     {
2718         if (m_jit && m_gpr != InvalidGPRReg)
2719             m_jit->unlock(gpr());
2720     }
2721
2722     GPRReg gpr()
2723     {
2724         return m_gpr;
2725     }
2726
2727 private:
2728     SpeculativeJIT* m_jit;
2729     GPRReg m_gpr;
2730 };
2731
2732 class JSValueRegsTemporary {
2733 public:
2734     JSValueRegsTemporary();
2735     JSValueRegsTemporary(SpeculativeJIT*);
2736     ~JSValueRegsTemporary();
2737
2738     JSValueRegs regs();
2739
2740 private:
2741 #if USE(JSVALUE64)
2742     GPRTemporary m_gpr;
2743 #else
2744     GPRTemporary m_payloadGPR;
2745     GPRTemporary m_tagGPR;
2746 #endif
2747 };
2748
2749 class FPRTemporary {
2750 public:
2751     FPRTemporary(SpeculativeJIT*);
2752     FPRTemporary(SpeculativeJIT*, SpeculateDoubleOperand&);
2753     FPRTemporary(SpeculativeJIT*, SpeculateDoubleOperand&, SpeculateDoubleOperand&);
2754 #if USE(JSVALUE32_64)
2755     FPRTemporary(SpeculativeJIT*, JSValueOperand&);
2756 #endif
2757
2758     ~FPRTemporary()
2759     {
2760         m_jit->unlock(fpr());
2761     }
2762
2763     FPRReg fpr() const
2764     {
2765         ASSERT(m_fpr != InvalidFPRReg);
2766         return m_fpr;
2767     }
2768
2769 protected:
2770     FPRTemporary(SpeculativeJIT* jit, FPRReg lockedFPR)
2771         : m_jit(jit)
2772         , m_fpr(lockedFPR)
2773     {
2774     }
2775
2776 private:
2777     SpeculativeJIT* m_jit;
2778     FPRReg m_fpr;
2779 };
2780
2781
2782 // === Results ===
2783 //
2784 // These classes lock the result of a call to a C++ helper function.
2785
2786 class GPRFlushedCallResult : public GPRTemporary {
2787 public:
2788     GPRFlushedCallResult(SpeculativeJIT* jit)
2789         : GPRTemporary(jit, GPRInfo::returnValueGPR)
2790     {
2791     }
2792 };
2793
2794 #if USE(JSVALUE32_64)
2795 class GPRFlushedCallResult2 : public GPRTemporary {
2796 public:
2797     GPRFlushedCallResult2(SpeculativeJIT* jit)
2798         : GPRTemporary(jit, GPRInfo::returnValueGPR2)
2799     {
2800     }
2801 };
2802 #endif
2803
2804 class FPRResult : public FPRTemporary {
2805 public:
2806     FPRResult(SpeculativeJIT* jit)
2807         : FPRTemporary(jit, lockedResult(jit))
2808     {
2809     }
2810
2811 private:
2812     static FPRReg lockedResult(SpeculativeJIT* jit)
2813     {
2814         jit->lock(FPRInfo::returnValueFPR);
2815         return FPRInfo::returnValueFPR;
2816     }
2817 };
2818
2819
2820 // === Speculative Operand types ===
2821 //
2822 // SpeculateInt32Operand, SpeculateStrictInt32Operand and SpeculateCellOperand.
2823 //
2824 // These are used to lock the operands to a node into machine registers within the
2825 // SpeculativeJIT. The classes operate like those above, however these will
2826 // perform a speculative check for a more restrictive type than we can statically
2827 // determine the operand to have. If the operand does not have the requested type,
2828 // a bail-out to the non-speculative path will be taken.
2829
2830 class SpeculateInt32Operand {
2831 public:
2832     explicit SpeculateInt32Operand(SpeculativeJIT* jit, Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2833         : m_jit(jit)
2834         , m_edge(edge)
2835         , m_gprOrInvalid(InvalidGPRReg)
2836 #ifndef NDEBUG
2837         , m_format(DataFormatNone)
2838 #endif
2839     {
2840         ASSERT(m_jit);
2841         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
2842         if (jit->isFilled(node()))
2843             gpr();
2844     }
2845
2846     ~SpeculateInt32Operand()
2847     {
2848         ASSERT(m_gprOrInvalid != InvalidGPRReg);
2849         m_jit->unlock(m_gprOrInvalid);
2850     }
2851
2852     Edge edge() const
2853     {
2854         return m_edge;
2855     }
2856
2857     Node* node() const
2858     {
2859         return edge().node();
2860     }
2861
2862     DataFormat format()
2863     {
2864         gpr(); // m_format is set when m_gpr is locked.
2865         ASSERT(m_format == DataFormatInt32 || m_format == DataFormatJSInt32);
2866         return m_format;
2867     }
2868
2869     GPRReg gpr()
2870     {
2871         if (m_gprOrInvalid == InvalidGPRReg)
2872             m_gprOrInvalid = m_jit->fillSpeculateInt32(edge(), m_format);
2873         return m_gprOrInvalid;
2874     }
2875
2876     void use()
2877     {
2878         m_jit->use(node());
2879     }
2880
2881 private:
2882     SpeculativeJIT* m_jit;
2883     Edge m_edge;
2884     GPRReg m_gprOrInvalid;
2885     DataFormat m_format;
2886 };
2887
2888 class SpeculateStrictInt32Operand {
2889 public:
2890     explicit SpeculateStrictInt32Operand(SpeculativeJIT* jit, Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
2891         : m_jit(jit)
2892         , m_edge(edge)
2893         , m_gprOrInvalid(InvalidGPRReg)
2894     {
2895         ASSERT(m_jit);
2896         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
2897         if (jit->isFilled(node()))
2898             gpr();
2899     }
2900
2901     ~SpeculateStrictInt32Operand()
2902     {
2903         ASSERT(m_gprOrInvalid != InvalidGPRReg);
2904         m_jit->unlock(m_gprOrInvalid);
2905     }
2906
2907     Edge edge() const
2908     {
2909         return m_edge;
2910     }
2911
2912     Node* node() const
2913     {
2914         return edge().node();
2915     }
2916
2917     GPRReg gpr()
2918     {
2919         if (m_gprOrInvalid == InvalidGPRReg)
2920             m_gprOrInvalid = m_jit->fillSpeculateInt32Strict(edge());
2921         return m_gprOrInvalid;
2922     }
2923
2924     void use()
2925     {
2926         m_jit->use(node());
2927     }
2928
2929 private:
2930     SpeculativeJIT* m_jit;
2931     Edge m_edge;
2932     GPRReg m_gprOrInvalid;
2933 };
2934
2935 // Gives you a canonical Int52 (i.e. it's left-shifted by 16, low bits zero).
2936 class SpeculateInt52Operand {
2937 public:
2938     explicit SpeculateInt52Operand(SpeculativeJIT* jit, Edge edge)
2939         : m_jit(jit)
2940         , m_edge(edge)
2941         , m_gprOrInvalid(InvalidGPRReg)
2942     {
2943         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
2944         if (jit->isFilled(node()))
2945             gpr();
2946     }
2947
2948     ~SpeculateInt52Operand()
2949     {
2950         ASSERT(m_gprOrInvalid != InvalidGPRReg);
2951         m_jit->unlock(m_gprOrInvalid);
2952     }
2953
2954     Edge edge() const
2955     {
2956         return m_edge;
2957     }
2958
2959     Node* node() const
2960     {
2961         return edge().node();
2962     }
2963
2964     GPRReg gpr()
2965     {
2966         if (m_gprOrInvalid == InvalidGPRReg)
2967             m_gprOrInvalid = m_jit->fillSpeculateInt52(edge(), DataFormatInt52);
2968         return m_gprOrInvalid;
2969     }
2970
2971     void use()
2972     {
2973         m_jit->use(node());
2974     }
2975
2976 private:
2977     SpeculativeJIT* m_jit;
2978     Edge m_edge;
2979     GPRReg m_gprOrInvalid;
2980 };
2981
2982 // Gives you a strict Int52 (i.e. the payload is in the low 48 bits, high 16 bits are sign-extended).
2983 class SpeculateStrictInt52Operand {
2984 public:
2985     explicit SpeculateStrictInt52Operand(SpeculativeJIT* jit, Edge edge)
2986         : m_jit(jit)
2987         , m_edge(edge)
2988         , m_gprOrInvalid(InvalidGPRReg)
2989     {
2990         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
2991         if (jit->isFilled(node()))
2992             gpr();
2993     }
2994
2995     ~SpeculateStrictInt52Operand()
2996     {
2997         ASSERT(m_gprOrInvalid != InvalidGPRReg);
2998         m_jit->unlock(m_gprOrInvalid);
2999     }
3000
3001     Edge edge() const
3002     {
3003         return m_edge;
3004     }
3005
3006     Node* node() const
3007     {
3008         return edge().node();
3009     }
3010
3011     GPRReg gpr()
3012     {
3013         if (m_gprOrInvalid == InvalidGPRReg)
3014             m_gprOrInvalid = m_jit->fillSpeculateInt52(edge(), DataFormatStrictInt52);
3015         return m_gprOrInvalid;
3016     }
3017
3018     void use()
3019     {
3020         m_jit->use(node());
3021     }
3022
3023 private:
3024     SpeculativeJIT* m_jit;
3025     Edge m_edge;
3026     GPRReg m_gprOrInvalid;
3027 };
3028
3029 enum OppositeShiftTag { OppositeShift };
3030
3031 class SpeculateWhicheverInt52Operand {
3032 public:
3033     explicit SpeculateWhicheverInt52Operand(SpeculativeJIT* jit, Edge edge)
3034         : m_jit(jit)
3035         , m_edge(edge)
3036         , m_gprOrInvalid(InvalidGPRReg)
3037         , m_strict(jit->betterUseStrictInt52(edge))
3038     {
3039         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
3040         if (jit->isFilled(node()))
3041             gpr();
3042     }
3043
3044     explicit SpeculateWhicheverInt52Operand(SpeculativeJIT* jit, Edge edge, const SpeculateWhicheverInt52Operand& other)
3045         : m_jit(jit)
3046         , m_edge(edge)
3047         , m_gprOrInvalid(InvalidGPRReg)
3048         , m_strict(other.m_strict)
3049     {
3050         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
3051         if (jit->isFilled(node()))
3052             gpr();
3053     }
3054
3055     explicit SpeculateWhicheverInt52Operand(SpeculativeJIT* jit, Edge edge, OppositeShiftTag, const SpeculateWhicheverInt52Operand& other)
3056         : m_jit(jit)
3057         , m_edge(edge)
3058         , m_gprOrInvalid(InvalidGPRReg)
3059         , m_strict(!other.m_strict)
3060     {
3061         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
3062         if (jit->isFilled(node()))
3063             gpr();
3064     }
3065
3066     ~SpeculateWhicheverInt52Operand()
3067     {
3068         ASSERT(m_gprOrInvalid != InvalidGPRReg);
3069         m_jit->unlock(m_gprOrInvalid);
3070     }
3071
3072     Edge edge() const
3073     {
3074         return m_edge;
3075     }
3076
3077     Node* node() const
3078     {
3079         return edge().node();
3080     }
3081
3082     GPRReg gpr()
3083     {
3084         if (m_gprOrInvalid == InvalidGPRReg) {
3085             m_gprOrInvalid = m_jit->fillSpeculateInt52(
3086                 edge(), m_strict ? DataFormatStrictInt52 : DataFormatInt52);
3087         }
3088         return m_gprOrInvalid;
3089     }
3090
3091     void use()
3092     {
3093         m_jit->use(node());
3094     }
3095
3096     DataFormat format() const
3097     {
3098         return m_strict ? DataFormatStrictInt52 : DataFormatInt52;
3099     }
3100
3101 private:
3102     SpeculativeJIT* m_jit;
3103     Edge m_edge;
3104     GPRReg m_gprOrInvalid;
3105     bool m_strict;
3106 };
3107
3108 class SpeculateDoubleOperand {
3109 public:
3110     explicit SpeculateDoubleOperand(SpeculativeJIT* jit, Edge edge)
3111         : m_jit(jit)
3112         , m_edge(edge)
3113         , m_fprOrInvalid(InvalidFPRReg)
3114     {
3115         ASSERT(m_jit);
3116         RELEASE_ASSERT(isDouble(edge.useKind()));
3117         if (jit->isFilled(node()))
3118             fpr();
3119     }
3120
3121     ~SpeculateDoubleOperand()
3122     {
3123         ASSERT(m_fprOrInvalid != InvalidFPRReg);
3124         m_jit->unlock(m_fprOrInvalid);
3125     }
3126
3127     Edge edge() const
3128     {
3129         return m_edge;
3130     }
3131
3132     Node* node() const
3133     {
3134         return edge().node();
3135     }
3136
3137     FPRReg fpr()
3138     {
3139         if (m_fprOrInvalid == InvalidFPRReg)
3140             m_fprOrInvalid = m_jit->fillSpeculateDouble(edge());
3141         return m_fprOrInvalid;
3142     }
3143
3144     void use()
3145     {
3146         m_jit->use(node());
3147     }
3148
3149 private:
3150     SpeculativeJIT* m_jit;
3151     Edge m_edge;
3152     FPRReg m_fprOrInvalid;
3153 };
3154
3155 class SpeculateCellOperand {
3156 public:
3157     explicit SpeculateCellOperand(SpeculativeJIT* jit, Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
3158         : m_jit(jit)
3159         , m_edge(edge)
3160         , m_gprOrInvalid(InvalidGPRReg)
3161     {
3162         ASSERT(m_jit);
3163         if (!edge)
3164             return;
3165         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isCell(edge.useKind()));
3166         if (jit->isFilled(node()))
3167             gpr();
3168     }
3169
3170     ~SpeculateCellOperand()
3171     {
3172         if (!m_edge)
3173             return;
3174         ASSERT(m_gprOrInvalid != InvalidGPRReg);
3175         m_jit->unlock(m_gprOrInvalid);
3176     }
3177
3178     Edge edge() const
3179     {
3180         return m_edge;
3181     }
3182
3183     Node* node() const
3184     {
3185         return edge().node();
3186     }
3187
3188     GPRReg gpr()
3189     {
3190         ASSERT(m_edge);
3191         if (m_gprOrInvalid == InvalidGPRReg)
3192             m_gprOrInvalid = m_jit->fillSpeculateCell(edge());
3193         return m_gprOrInvalid;
3194     }
3195
3196     void use()
3197     {
3198         ASSERT(m_edge);
3199         m_jit->use(node());
3200     }
3201
3202 private:
3203     SpeculativeJIT* m_jit;
3204     Edge m_edge;
3205     GPRReg m_gprOrInvalid;
3206 };
3207
3208 class SpeculateBooleanOperand {
3209 public:
3210     explicit SpeculateBooleanOperand(SpeculativeJIT* jit, Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
3211         : m_jit(jit)
3212         , m_edge(edge)
3213         , m_gprOrInvalid(InvalidGPRReg)
3214     {
3215         ASSERT(m_jit);
3216         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
3217         if (jit->isFilled(node()))
3218             gpr();
3219     }
3220
3221     ~SpeculateBooleanOperand()
3222     {
3223         ASSERT(m_gprOrInvalid != InvalidGPRReg);
3224         m_jit->unlock(m_gprOrInvalid);
3225     }
3226
3227     Edge edge() const
3228     {
3229         return m_edge;
3230     }
3231
3232     Node* node() const
3233     {
3234         return edge().node();
3235     }
3236
3237     GPRReg gpr()
3238     {
3239         if (m_gprOrInvalid == InvalidGPRReg)
3240             m_gprOrInvalid = m_jit->fillSpeculateBoolean(edge());
3241         return m_gprOrInvalid;
3242     }
3243
3244     void use()
3245     {
3246         m_jit->use(node());
3247     }
3248
3249 private:
3250     SpeculativeJIT* m_jit;
3251     Edge m_edge;
3252     GPRReg m_gprOrInvalid;
3253 };
3254
3255 template<typename StructureLocationType>
3256 void SpeculativeJIT::speculateStringObjectForStructure(Edge edge, StructureLocationType structureLocation)
3257 {
3258     Structure* stringObjectStructure =
3259         m_jit.globalObjectFor(m_currentNode->origin.semantic)->stringObjectStructure();
3260
3261     if (!m_state.forNode(edge).m_structure.isSubsetOf(StructureSet(stringObjectStructure))) {
3262         speculationCheck(
3263             NotStringObject, JSValueRegs(), 0,
3264             m_jit.branchStructurePtr(
3265                 JITCompiler::NotEqual, structureLocation, stringObjectStructure));
3266     }
3267 }
3268
3269 #define DFG_TYPE_CHECK(source, edge, typesPassedThrough, jumpToFail) do { \
3270         JSValueSource _dtc_source = (source);                           \
3271         Edge _dtc_edge = (edge);                                        \
3272         SpeculatedType _dtc_typesPassedThrough = typesPassedThrough;    \
3273         if (!needsTypeCheck(_dtc_edge, _dtc_typesPassedThrough))        \
3274             break;                                                      \
3275         typeCheck(_dtc_source, _dtc_edge, _dtc_typesPassedThrough, (jumpToFail)); \
3276     } while (0)
3277
3278 } } // namespace JSC::DFG
3279
3280 #endif
3281 #endif
3282