ftl/FTLLowerDFGToLLVM.cpp

   1 /*
   2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToLLVM.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "CodeBlockWithJITType.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "FTLAbstractHeapRepository.h"
  35 #include "FTLAvailableRecovery.h"
  36 #include "FTLForOSREntryJITCode.h"
  37 #include "FTLFormattedValue.h"
  38 #include "FTLInlineCacheSize.h"
  39 #include "FTLLoweredNodeValue.h"
  40 #include "FTLOutput.h"
  41 #include "FTLThunks.h"
  42 #include "FTLWeightedTarget.h"
  43 #include "OperandsInlines.h"
  44 #include "JSCInlines.h"
  45 #include "VirtualRegister.h"
  46 #include <atomic>
  47 #include <wtf/ProcessID.h>
  48
  49 namespace JSC { namespace FTL {
  50
  51 using namespace DFG;
  52
  53 static std::atomic<int> compileCounter;
  54
  55 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
  56 // significantly less dead code.
  57 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
  58         FormattedValue _ftc_lowValue = (lowValue);                      \
  59         Edge _ftc_highValue = (highValue);                              \
  60         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
  61         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
  62             break;                                                      \
  63         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
  64     } while (false)
  65
  66 class LowerDFGToLLVM {
  67 public:
  68     LowerDFGToLLVM(State& state)
  69         : m_graph(state.graph)
  70         , m_ftlState(state)
  71         , m_loweringSucceeded(true)
  72         , m_heaps(state.context)
  73         , m_out(state.context)
  74         , m_availability(OperandsLike, state.graph.block(0)->variablesAtHead)
  75         , m_state(state.graph)
  76         , m_interpreter(state.graph, m_state)
  77         , m_stackmapIDs(0)
  78     {
  79     }
  80
  81
  82 #define LOWERING_FAILED(node, reason)                                  \
  83     loweringFailed((node), __FILE__, __LINE__, WTF_PRETTY_FUNCTION, (reason));
  84
  85     bool lower()
  86     {
  87         CString name;
  88         if (verboseCompilationEnabled()) {
  89             name = toCString(
  90                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
  91                 "_", codeBlock()->hash());
  92         } else
  93             name = "jsBody";
  94
  95         m_graph.m_dominators.computeIfNecessary(m_graph);
  96
  97         m_ftlState.module =
  98             llvm->ModuleCreateWithNameInContext(name.data(), m_ftlState.context);
  99
 100         m_ftlState.function = addFunction(
 101             m_ftlState.module, name.data(), functionType(m_out.int64));
 102         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
 103         if (isX86() && Options::llvmDisallowAVX()) {
 104             // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x
 105             // slower. It should be disabled.
 106             addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");
 107         }
 108
 109         if (verboseCompilationEnabled())
 110             dataLog("Function ready, beginning lowering.\n");
 111
 112         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
 113
 114         m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));
 115         LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));
 116         m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));
 117
 118         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 119             m_highBlock = m_graph.block(blockIndex);
 120             if (!m_highBlock)
 121                 continue;
 122             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
 123         }
 124
 125         m_out.appendTo(m_prologue, stackOverflow);
 126         createPhiVariables();
 127         LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));
 128         m_captured = m_out.add(
 129             m_out.ptrToInt(capturedAlloca, m_out.intPtr),
 130             m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));
 131
 132         // We should not create any alloca's after this point, since they will cease to
 133         // be mem2reg candidates.
 134
 135         m_ftlState.capturedStackmapID = m_stackmapIDs++;
 136         m_out.call(
 137             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),
 138             m_out.int32Zero, capturedAlloca);
 139
 140         m_callFrame = m_out.ptrToInt(
 141             m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);
 142         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 143         m_tagMask = m_out.constInt64(TagMask);
 144
 145         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
 146
 147         m_out.branch(
 148             didOverflowStack(), rarely(stackOverflow), usually(lowBlock(m_graph.block(0))));
 149
 150         m_out.appendTo(stackOverflow, m_handleExceptions);
 151         m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
 152         m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;
 153         m_out.call(
 154             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),
 155             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 156         m_out.unreachable();
 157
 158         m_out.appendTo(m_handleExceptions, lowBlock(m_graph.block(0)));
 159         m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;
 160         m_out.call(
 161             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),
 162             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 163         m_out.unreachable();
 164
 165         if (!m_loweringSucceeded)
 166             return m_loweringSucceeded;
 167
 168         Vector<BasicBlock*> depthFirst;
 169         m_graph.getBlocksInDepthFirstOrder(depthFirst);
 170         for (unsigned i = 0; i < depthFirst.size(); ++i) {
 171             compileBlock(depthFirst[i]);
 172             if (!m_loweringSucceeded)
 173                 return m_loweringSucceeded;
 174         }
 175
 176         if (Options::dumpLLVMIR())
 177             dumpModule(m_ftlState.module);
 178
 179         if (verboseCompilationEnabled())
 180             m_ftlState.dumpState("after lowering");
 181         if (validationEnabled())
 182             verifyModule(m_ftlState.module);
 183
 184         return m_loweringSucceeded;
 185     }
 186
 187 private:
 188
 189     void createPhiVariables()
 190     {
 191         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 192             BasicBlock* block = m_graph.block(blockIndex);
 193             if (!block)
 194                 continue;
 195             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 196                 Node* node = block->at(nodeIndex);
 197                 if (node->op() != Phi)
 198                     continue;
 199                 LType type;
 200                 switch (node->flags() & NodeResultMask) {
 201                 case NodeResultDouble:
 202                     type = m_out.doubleType;
 203                     break;
 204                 case NodeResultInt32:
 205                     type = m_out.int32;
 206                     break;
 207                 case NodeResultInt52:
 208                     type = m_out.int64;
 209                     break;
 210                 case NodeResultBoolean:
 211                     type = m_out.boolean;
 212                     break;
 213                 case NodeResultJS:
 214                     type = m_out.int64;
 215                     break;
 216                 default:
 217                     LOWERING_FAILED(node, "Bad Phi node result type");
 218                     return;
 219                 }
 220                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
 221             }
 222         }
 223     }
 224
 225     void compileBlock(BasicBlock* block)
 226     {
 227         if (!block)
 228             return;
 229
 230         if (verboseCompilationEnabled())
 231             dataLog("Compiling block ", *block, "\n");
 232
 233         m_highBlock = block;
 234
 235         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 236
 237         m_nextHighBlock = 0;
 238         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 239             m_nextHighBlock = m_graph.block(nextBlockIndex);
 240             if (m_nextHighBlock)
 241                 break;
 242         }
 243         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 244
 245         // All of this effort to find the next block gives us the ability to keep the
 246         // generated IR in roughly program order. This ought not affect the performance
 247         // of the generated code (since we expect LLVM to reorder things) but it will
 248         // make IR dumps easier to read.
 249         m_out.appendTo(lowBlock, m_nextLowBlock);
 250
 251         if (Options::ftlCrashes())
 252             m_out.crashNonTerminal();
 253
 254         if (!m_highBlock->cfaHasVisited) {
 255             m_out.crash();
 256             return;
 257         }
 258
 259         initializeOSRExitStateForBlock();
 260
 261         m_state.reset();
 262         m_state.beginBasicBlock(m_highBlock);
 263
 264         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 265             if (!compileNode(m_nodeIndex))
 266                 break;
 267         }
 268     }
 269
 270     bool compileNode(unsigned nodeIndex)
 271     {
 272         if (!m_state.isValid()) {
 273             m_out.unreachable();
 274             return false;
 275         }
 276
 277         m_node = m_highBlock->at(nodeIndex);
 278         m_codeOriginForExitProfile = m_node->origin.semantic;
 279         m_codeOriginForExitTarget = m_node->origin.forExit;
 280
 281         if (verboseCompilationEnabled())
 282             dataLog("Lowering ", m_node, "\n");
 283
 284         m_availableRecoveries.resize(0);
 285
 286         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
 287
 288         switch (m_node->op()) {
 289         case Upsilon:
 290             compileUpsilon();
 291             break;
 292         case Phi:
 293             compilePhi();
 294             break;
 295         case JSConstant:
 296             break;
 297         case DoubleConstant:
 298             compileDoubleConstant();
 299             break;
 300         case Int52Constant:
 301             compileInt52Constant();
 302             break;
 303         case WeakJSConstant:
 304             compileWeakJSConstant();
 305             break;
 306         case PhantomArguments:
 307             compilePhantomArguments();
 308             break;
 309         case DoubleRep:
 310             compileDoubleRep();
 311             break;
 312         case ValueRep:
 313             compileValueRep();
 314             break;
 315         case Int52Rep:
 316             compileInt52Rep();
 317             break;
 318         case ValueToInt32:
 319             compileValueToInt32();
 320             break;
 321         case BooleanToNumber:
 322             compileBooleanToNumber();
 323             break;
 324         case GetArgument:
 325             compileGetArgument();
 326             break;
 327         case ExtractOSREntryLocal:
 328             compileExtractOSREntryLocal();
 329             break;
 330         case GetLocal:
 331             compileGetLocal();
 332             break;
 333         case SetLocal:
 334             compileSetLocal();
 335             break;
 336         case MovHint:
 337             compileMovHint();
 338             break;
 339         case GetMyArgumentsLength:
 340             compileGetMyArgumentsLength();
 341             break;
 342         case GetMyArgumentByVal:
 343             compileGetMyArgumentByVal();
 344             break;
 345         case ZombieHint:
 346             compileZombieHint();
 347             break;
 348         case Phantom:
 349         case HardPhantom:
 350             compilePhantom();
 351             break;
 352         case ToThis:
 353             compileToThis();
 354             break;
 355         case ValueAdd:
 356             compileValueAdd();
 357             break;
 358         case ArithAdd:
 359         case ArithSub:
 360             compileArithAddOrSub();
 361             break;
 362         case ArithMul:
 363             compileArithMul();
 364             break;
 365         case ArithDiv:
 366             compileArithDiv();
 367             break;
 368         case ArithMod:
 369             compileArithMod();
 370             break;
 371         case ArithMin:
 372         case ArithMax:
 373             compileArithMinOrMax();
 374             break;
 375         case ArithAbs:
 376             compileArithAbs();
 377             break;
 378         case ArithSin:
 379             compileArithSin();
 380             break;
 381         case ArithCos:
 382             compileArithCos();
 383             break;
 384         case ArithSqrt:
 385             compileArithSqrt();
 386             break;
 387         case ArithFRound:
 388             compileArithFRound();
 389             break;
 390         case ArithNegate:
 391             compileArithNegate();
 392             break;
 393         case BitAnd:
 394             compileBitAnd();
 395             break;
 396         case BitOr:
 397             compileBitOr();
 398             break;
 399         case BitXor:
 400             compileBitXor();
 401             break;
 402         case BitRShift:
 403             compileBitRShift();
 404             break;
 405         case BitLShift:
 406             compileBitLShift();
 407             break;
 408         case BitURShift:
 409             compileBitURShift();
 410             break;
 411         case UInt32ToNumber:
 412             compileUInt32ToNumber();
 413             break;
 414         case CheckStructure:
 415             compileCheckStructure();
 416             break;
 417         case StructureTransitionWatchpoint:
 418             compileStructureTransitionWatchpoint();
 419             break;
 420         case CheckFunction:
 421             compileCheckFunction();
 422             break;
 423         case CheckExecutable:
 424             compileCheckExecutable();
 425             break;
 426         case ArrayifyToStructure:
 427             compileArrayifyToStructure();
 428             break;
 429         case PutStructure:
 430             compilePutStructure();
 431             break;
 432         case PhantomPutStructure:
 433             compilePhantomPutStructure();
 434             break;
 435         case GetById:
 436             compileGetById();
 437             break;
 438         case PutByIdDirect:
 439         case PutById:
 440             compilePutById();
 441             break;
 442         case GetButterfly:
 443             compileGetButterfly();
 444             break;
 445         case ConstantStoragePointer:
 446             compileConstantStoragePointer();
 447             break;
 448         case GetIndexedPropertyStorage:
 449             compileGetIndexedPropertyStorage();
 450             break;
 451         case CheckArray:
 452             compileCheckArray();
 453             break;
 454         case GetArrayLength:
 455             compileGetArrayLength();
 456             break;
 457         case CheckInBounds:
 458             compileCheckInBounds();
 459             break;
 460         case GetByVal:
 461             compileGetByVal();
 462             break;
 463         case PutByVal:
 464         case PutByValAlias:
 465         case PutByValDirect:
 466             compilePutByVal();
 467             break;
 468         case ArrayPush:
 469             compileArrayPush();
 470             break;
 471         case ArrayPop:
 472             compileArrayPop();
 473             break;
 474         case NewObject:
 475             compileNewObject();
 476             break;
 477         case NewArray:
 478             compileNewArray();
 479             break;
 480         case NewArrayBuffer:
 481             compileNewArrayBuffer();
 482             break;
 483         case NewArrayWithSize:
 484             compileNewArrayWithSize();
 485             break;
 486         case GetTypedArrayByteOffset:
 487             compileGetTypedArrayByteOffset();
 488             break;
 489         case AllocatePropertyStorage:
 490             compileAllocatePropertyStorage();
 491             break;
 492         case ReallocatePropertyStorage:
 493             compileReallocatePropertyStorage();
 494             break;
 495         case ToString:
 496             compileToString();
 497             break;
 498         case ToPrimitive:
 499             compileToPrimitive();
 500             break;
 501         case MakeRope:
 502             compileMakeRope();
 503             break;
 504         case StringCharAt:
 505             compileStringCharAt();
 506             break;
 507         case StringCharCodeAt:
 508             compileStringCharCodeAt();
 509             break;
 510         case GetByOffset:
 511             compileGetByOffset();
 512             break;
 513         case MultiGetByOffset:
 514             compileMultiGetByOffset();
 515             break;
 516         case PutByOffset:
 517             compilePutByOffset();
 518             break;
 519         case MultiPutByOffset:
 520             compileMultiPutByOffset();
 521             break;
 522         case GetGlobalVar:
 523             compileGetGlobalVar();
 524             break;
 525         case PutGlobalVar:
 526             compilePutGlobalVar();
 527             break;
 528         case NotifyWrite:
 529             compileNotifyWrite();
 530             break;
 531         case GetCallee:
 532             compileGetCallee();
 533             break;
 534         case GetScope:
 535             compileGetScope();
 536             break;
 537         case GetMyScope:
 538             compileGetMyScope();
 539             break;
 540         case SkipScope:
 541             compileSkipScope();
 542             break;
 543         case GetClosureRegisters:
 544             compileGetClosureRegisters();
 545             break;
 546         case GetClosureVar:
 547             compileGetClosureVar();
 548             break;
 549         case PutClosureVar:
 550             compilePutClosureVar();
 551             break;
 552         case CompareEq:
 553             compileCompareEq();
 554             break;
 555         case CompareEqConstant:
 556             compileCompareEqConstant();
 557             break;
 558         case CompareStrictEq:
 559             compileCompareStrictEq();
 560             break;
 561         case CompareLess:
 562             compileCompareLess();
 563             break;
 564         case CompareLessEq:
 565             compileCompareLessEq();
 566             break;
 567         case CompareGreater:
 568             compileCompareGreater();
 569             break;
 570         case CompareGreaterEq:
 571             compileCompareGreaterEq();
 572             break;
 573         case LogicalNot:
 574             compileLogicalNot();
 575             break;
 576         case Call:
 577         case Construct:
 578             compileCallOrConstruct();
 579             break;
 580         case Jump:
 581             compileJump();
 582             break;
 583         case Branch:
 584             compileBranch();
 585             break;
 586         case Switch:
 587             compileSwitch();
 588             break;
 589         case Return:
 590             compileReturn();
 591             break;
 592         case ForceOSRExit:
 593             compileForceOSRExit();
 594             break;
 595         case Throw:
 596         case ThrowReferenceError:
 597             compileThrow();
 598             break;
 599         case InvalidationPoint:
 600             compileInvalidationPoint();
 601             break;
 602         case CheckArgumentsNotCreated:
 603             compileCheckArgumentsNotCreated();
 604             break;
 605         case IsUndefined:
 606             compileIsUndefined();
 607             break;
 608         case IsBoolean:
 609             compileIsBoolean();
 610             break;
 611         case IsNumber:
 612             compileIsNumber();
 613             break;
 614         case IsString:
 615             compileIsString();
 616             break;
 617         case IsObject:
 618             compileIsObject();
 619             break;
 620         case IsFunction:
 621             compileIsFunction();
 622             break;
 623         case CheckHasInstance:
 624             compileCheckHasInstance();
 625             break;
 626         case InstanceOf:
 627             compileInstanceOf();
 628             break;
 629         case CountExecution:
 630             compileCountExecution();
 631             break;
 632         case StoreBarrier:
 633             compileStoreBarrier();
 634             break;
 635         case StoreBarrierWithNullCheck:
 636             compileStoreBarrierWithNullCheck();
 637             break;
 638         case PhantomLocal:
 639         case SetArgument:
 640         case LoopHint:
 641         case VariableWatchpoint:
 642         case FunctionReentryWatchpoint:
 643         case TypedArrayWatchpoint:
 644         case AllocationProfileWatchpoint:
 645             break;
 646         default:
 647             LOWERING_FAILED(m_node, "Unrecognized node in FTL backend");
 648             break;
 649         }
 650
 651         if (!m_loweringSucceeded)
 652             return false;
 653
 654         if (shouldExecuteEffects)
 655             m_interpreter.executeEffects(nodeIndex);
 656
 657         return true;
 658     }
 659
 660     void compileUpsilon()
 661     {
 662         LValue destination = m_phis.get(m_node->phi());
 663
 664         switch (m_node->child1().useKind()) {
 665         case DoubleRepUse:
 666             m_out.set(lowDouble(m_node->child1()), destination);
 667             break;
 668         case Int32Use:
 669             m_out.set(lowInt32(m_node->child1()), destination);
 670             break;
 671         case Int52RepUse:
 672             m_out.set(lowInt52(m_node->child1()), destination);
 673             break;
 674         case BooleanUse:
 675             m_out.set(lowBoolean(m_node->child1()), destination);
 676             break;
 677         case CellUse:
 678             m_out.set(lowCell(m_node->child1()), destination);
 679             break;
 680         case UntypedUse:
 681             m_out.set(lowJSValue(m_node->child1()), destination);
 682             break;
 683         default:
 684             LOWERING_FAILED(m_node, "Bad use kind");
 685             break;
 686         }
 687     }
 688
 689     void compilePhi()
 690     {
 691         LValue source = m_phis.get(m_node);
 692
 693         switch (m_node->flags() & NodeResultMask) {
 694         case NodeResultDouble:
 695             setDouble(m_out.get(source));
 696             break;
 697         case NodeResultInt32:
 698             setInt32(m_out.get(source));
 699             break;
 700         case NodeResultInt52:
 701             setInt52(m_out.get(source));
 702             break;
 703         case NodeResultBoolean:
 704             setBoolean(m_out.get(source));
 705             break;
 706         case NodeResultJS:
 707             setJSValue(m_out.get(source));
 708             break;
 709         default:
 710             LOWERING_FAILED(m_node, "Bad use kind");
 711             break;
 712         }
 713     }
 714
 715     void compileDoubleConstant()
 716     {
 717         setDouble(m_out.constDouble(m_graph.valueOfNumberConstant(m_node)));
 718     }
 719
 720     void compileInt52Constant()
 721     {
 722         int64_t value = m_graph.valueOfJSConstant(m_node).asMachineInt();
 723
 724         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
 725         setStrictInt52(m_out.constInt64(value));
 726     }
 727
 728     void compileWeakJSConstant()
 729     {
 730         setJSValue(weakPointer(m_node->weakConstant()));
 731     }
 732
 733     void compilePhantomArguments()
 734     {
 735         setJSValue(m_out.constInt64(JSValue::encode(JSValue())));
 736     }
 737
 738     void compileDoubleRep()
 739     {
 740         switch (m_node->child1().useKind()) {
 741         case NumberUse: {
 742             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
 743             setDouble(jsValueToDouble(m_node->child1(), value));
 744             return;
 745         }
 746
 747         case Int52RepUse: {
 748             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
 749             return;
 750         }
 751
 752         default:
 753             LOWERING_FAILED(m_node, "Bad use kind");
 754         }
 755     }
 756
 757     void compileValueRep()
 758     {
 759         switch (m_node->child1().useKind()) {
 760         case DoubleRepUse: {
 761             LValue value = lowDouble(m_node->child1());
 762
 763             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
 764                 value = m_out.select(
 765                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
 766             }
 767
 768             setJSValue(boxDouble(value));
 769             return;
 770         }
 771
 772         case Int52RepUse: {
 773             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
 774             return;
 775         }
 776
 777         default:
 778             LOWERING_FAILED(m_node, "Bad use kind");
 779         }
 780     }
 781
 782     void compileInt52Rep()
 783     {
 784         switch (m_node->child1().useKind()) {
 785         case Int32Use:
 786             setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64));
 787             return;
 788
 789         case MachineIntUse:
 790             setStrictInt52(
 791                 jsValueToStrictInt52(
 792                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
 793             return;
 794
 795         case DoubleRepMachineIntUse:
 796             setStrictInt52(
 797                 doubleToStrictInt52(
 798                     m_node->child1(), lowDouble(m_node->child1())));
 799             return;
 800
 801         default:
 802             LOWERING_FAILED(m_node, "Bad use kind");
 803         }
 804     }
 805
 806     void compileValueToInt32()
 807     {
 808         switch (m_node->child1().useKind()) {
 809         case Int52RepUse:
 810             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
 811             break;
 812
 813         case DoubleRepUse:
 814             setInt32(doubleToInt32(lowDouble(m_node->child1())));
 815             break;
 816
 817         case NumberUse:
 818         case NotCellUse: {
 819             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
 820             if (isValid(value)) {
 821                 setInt32(value.value());
 822                 break;
 823             }
 824
 825             value = m_jsValueValues.get(m_node->child1().node());
 826             if (isValid(value)) {
 827                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
 828                 break;
 829             }
 830
 831             // We'll basically just get here for constants. But it's good to have this
 832             // catch-all since we often add new representations into the mix.
 833             setInt32(
 834                 numberOrNotCellToInt32(
 835                     m_node->child1(),
 836                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
 837             break;
 838         }
 839
 840         default:
 841             LOWERING_FAILED(m_node, "Bad use kind");
 842             break;
 843         }
 844     }
 845
 846     void compileBooleanToNumber()
 847     {
 848         switch (m_node->child1().useKind()) {
 849         case BooleanUse: {
 850             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
 851             return;
 852         }
 853
 854         case UntypedUse: {
 855             LValue value = lowJSValue(m_node->child1());
 856
 857             LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case"));
 858             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation"));
 859
 860             ValueFromBlock notBooleanResult = m_out.anchor(value);
 861             m_out.branch(isBoolean(value), unsure(booleanCase), unsure(continuation));
 862
 863             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
 864             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
 865                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
 866             m_out.jump(continuation);
 867
 868             m_out.appendTo(continuation, lastNext);
 869             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
 870             return;
 871         }
 872
 873         default:
 874             LOWERING_FAILED(m_node, "Bad flush format");
 875             return;
 876         }
 877     }
 878
 879     void compileGetArgument()
 880     {
 881         VariableAccessData* variable = m_node->variableAccessData();
 882         VirtualRegister operand = variable->machineLocal();
 883         RELEASE_ASSERT(operand.isArgument());
 884
 885         LValue jsValue = m_out.load64(addressFor(operand));
 886
 887         switch (useKindFor(variable->flushFormat())) {
 888         case Int32Use:
 889             speculate(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
 890             setInt32(unboxInt32(jsValue));
 891             break;
 892         case CellUse:
 893             speculate(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
 894             setJSValue(jsValue);
 895             break;
 896         case BooleanUse:
 897             speculate(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
 898             setBoolean(unboxBoolean(jsValue));
 899             break;
 900         case UntypedUse:
 901             setJSValue(jsValue);
 902             break;
 903         default:
 904             LOWERING_FAILED(m_node, "Bad use kind");
 905             break;
 906         }
 907     }
 908
 909     void compileExtractOSREntryLocal()
 910     {
 911         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
 912             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
 913         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
 914     }
 915
 916     void compileGetLocal()
 917     {
 918         // GetLocals arise only for captured variables.
 919
 920         VariableAccessData* variable = m_node->variableAccessData();
 921         AbstractValue& value = m_state.variables().operand(variable->local());
 922
 923         RELEASE_ASSERT(variable->isCaptured());
 924
 925         if (isInt32Speculation(value.m_type))
 926             setInt32(m_out.load32(payloadFor(variable->machineLocal())));
 927         else
 928             setJSValue(m_out.load64(addressFor(variable->machineLocal())));
 929     }
 930
 931     void compileSetLocal()
 932     {
 933         VariableAccessData* variable = m_node->variableAccessData();
 934         switch (variable->flushFormat()) {
 935         case FlushedJSValue:
 936         case FlushedArguments: {
 937             LValue value = lowJSValue(m_node->child1());
 938             m_out.store64(value, addressFor(variable->machineLocal()));
 939             break;
 940         }
 941
 942         case FlushedDouble: {
 943             LValue value = lowDouble(m_node->child1());
 944             m_out.storeDouble(value, addressFor(variable->machineLocal()));
 945             break;
 946         }
 947
 948         case FlushedInt32: {
 949             LValue value = lowInt32(m_node->child1());
 950             m_out.store32(value, payloadFor(variable->machineLocal()));
 951             break;
 952         }
 953
 954         case FlushedInt52: {
 955             LValue value = lowInt52(m_node->child1());
 956             m_out.store64(value, addressFor(variable->machineLocal()));
 957             break;
 958         }
 959
 960         case FlushedCell: {
 961             LValue value = lowCell(m_node->child1());
 962             m_out.store64(value, addressFor(variable->machineLocal()));
 963             break;
 964         }
 965
 966         case FlushedBoolean: {
 967             speculateBoolean(m_node->child1());
 968             m_out.store64(
 969                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
 970                 addressFor(variable->machineLocal()));
 971             break;
 972         }
 973
 974         default:
 975             LOWERING_FAILED(m_node, "Bad flush format for argument");
 976             return;
 977         }
 978
 979         m_availability.operand(variable->local()) = Availability(variable->flushedAt());
 980     }
 981
 982     void compileMovHint()
 983     {
 984         ASSERT(m_node->containsMovHint());
 985         ASSERT(m_node->op() != ZombieHint);
 986
 987         VirtualRegister operand = m_node->unlinkedLocal();
 988         m_availability.operand(operand) = Availability(m_node->child1().node());
 989     }
 990
 991     void compileZombieHint()
 992     {
 993         m_availability.operand(m_node->unlinkedLocal()) = Availability::unavailable();
 994     }
 995
 996     void compilePhantom()
 997     {
 998         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
 999     }
1000
1001     void compileToThis()
1002     {
1003         LValue value = lowJSValue(m_node->child1());
1004
1005         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case"));
1006         LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case"));
1007         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation"));
1008
1009         m_out.branch(isCell(value), usually(isCellCase), rarely(slowCase));
1010
1011         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1012         ValueFromBlock fastResult = m_out.anchor(value);
1013         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1014
1015         m_out.appendTo(slowCase, continuation);
1016         J_JITOperation_EJ function;
1017         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1018             function = operationToThisStrict;
1019         else
1020             function = operationToThis;
1021         ValueFromBlock slowResult = m_out.anchor(
1022             vmCall(m_out.operation(function), m_callFrame, value));
1023         m_out.jump(continuation);
1024
1025         m_out.appendTo(continuation, lastNext);
1026         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1027     }
1028
1029     void compileValueAdd()
1030     {
1031         J_JITOperation_EJJ operation;
1032         if (!(m_state.forNode(m_node->child1()).m_type & SpecFullNumber)
1033             && !(m_state.forNode(m_node->child2()).m_type & SpecFullNumber))
1034             operation = operationValueAddNotNumber;
1035         else
1036             operation = operationValueAdd;
1037         setJSValue(vmCall(
1038             m_out.operation(operation), m_callFrame,
1039             lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
1040     }
1041
1042     void compileArithAddOrSub()
1043     {
1044         bool isSub =  m_node->op() == ArithSub;
1045         switch (m_node->binaryUseKind()) {
1046         case Int32Use: {
1047             LValue left = lowInt32(m_node->child1());
1048             LValue right = lowInt32(m_node->child2());
1049
1050             if (!shouldCheckOverflow(m_node->arithMode())) {
1051                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1052                 break;
1053             }
1054
1055             LValue result;
1056             if (!isSub) {
1057                 result = m_out.addWithOverflow32(left, right);
1058
1059                 if (doesKill(m_node->child2())) {
1060                     addAvailableRecovery(
1061                         m_node->child2(), SubRecovery,
1062                         m_out.extractValue(result, 0), left, ValueFormatInt32);
1063                 } else if (doesKill(m_node->child1())) {
1064                     addAvailableRecovery(
1065                         m_node->child1(), SubRecovery,
1066                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1067                 }
1068             } else {
1069                 result = m_out.subWithOverflow32(left, right);
1070
1071                 if (doesKill(m_node->child2())) {
1072                     // result = left - right
1073                     // result - left = -right
1074                     // right = left - result
1075                     addAvailableRecovery(
1076                         m_node->child2(), SubRecovery,
1077                         left, m_out.extractValue(result, 0), ValueFormatInt32);
1078                 } else if (doesKill(m_node->child1())) {
1079                     // result = left - right
1080                     // result + right = left
1081                     addAvailableRecovery(
1082                         m_node->child1(), AddRecovery,
1083                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1084                 }
1085             }
1086
1087             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
1088             setInt32(m_out.extractValue(result, 0));
1089             break;
1090         }
1091
1092         case Int52RepUse: {
1093             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)
1094                 && !m_state.forNode(m_node->child2()).couldBeType(SpecInt52)) {
1095                 Int52Kind kind;
1096                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1097                 LValue right = lowInt52(m_node->child2(), kind);
1098                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1099                 break;
1100             }
1101
1102             LValue left = lowInt52(m_node->child1());
1103             LValue right = lowInt52(m_node->child2());
1104
1105             LValue result;
1106             if (!isSub) {
1107                 result = m_out.addWithOverflow64(left, right);
1108
1109                 if (doesKill(m_node->child2())) {
1110                     addAvailableRecovery(
1111                         m_node->child2(), SubRecovery,
1112                         m_out.extractValue(result, 0), left, ValueFormatInt52);
1113                 } else if (doesKill(m_node->child1())) {
1114                     addAvailableRecovery(
1115                         m_node->child1(), SubRecovery,
1116                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1117                 }
1118             } else {
1119                 result = m_out.subWithOverflow64(left, right);
1120
1121                 if (doesKill(m_node->child2())) {
1122                     // result = left - right
1123                     // result - left = -right
1124                     // right = left - result
1125                     addAvailableRecovery(
1126                         m_node->child2(), SubRecovery,
1127                         left, m_out.extractValue(result, 0), ValueFormatInt52);
1128                 } else if (doesKill(m_node->child1())) {
1129                     // result = left - right
1130                     // result + right = left
1131                     addAvailableRecovery(
1132                         m_node->child1(), AddRecovery,
1133                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1134                 }
1135             }
1136
1137             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
1138             setInt52(m_out.extractValue(result, 0));
1139             break;
1140         }
1141
1142         case DoubleRepUse: {
1143             LValue C1 = lowDouble(m_node->child1());
1144             LValue C2 = lowDouble(m_node->child2());
1145
1146             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1147             break;
1148         }
1149
1150         default:
1151             LOWERING_FAILED(m_node, "Bad use kind");
1152             break;
1153         }
1154     }
1155
1156     void compileArithMul()
1157     {
1158         switch (m_node->binaryUseKind()) {
1159         case Int32Use: {
1160             LValue left = lowInt32(m_node->child1());
1161             LValue right = lowInt32(m_node->child2());
1162
1163             LValue result;
1164
1165             if (!shouldCheckOverflow(m_node->arithMode()))
1166                 result = m_out.mul(left, right);
1167             else {
1168                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
1169                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1170                 result = m_out.extractValue(overflowResult, 0);
1171             }
1172
1173             if (shouldCheckNegativeZero(m_node->arithMode())) {
1174                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1175                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1176
1177                 m_out.branch(
1178                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1179
1180                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1181                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero));
1182                 speculate(NegativeZero, noValue(), 0, cond);
1183                 m_out.jump(continuation);
1184                 m_out.appendTo(continuation, lastNext);
1185             }
1186
1187             setInt32(result);
1188             break;
1189         }
1190
1191         case Int52RepUse: {
1192             Int52Kind kind;
1193             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1194             LValue right = lowInt52(m_node->child2(), opposite(kind));
1195
1196             LValue overflowResult = m_out.mulWithOverflow64(left, right);
1197             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1198             LValue result = m_out.extractValue(overflowResult, 0);
1199
1200             if (shouldCheckNegativeZero(m_node->arithMode())) {
1201                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1202                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1203
1204                 m_out.branch(
1205                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1206
1207                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1208                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero));
1209                 speculate(NegativeZero, noValue(), 0, cond);
1210                 m_out.jump(continuation);
1211                 m_out.appendTo(continuation, lastNext);
1212             }
1213
1214             setInt52(result);
1215             break;
1216         }
1217
1218         case DoubleRepUse: {
1219             setDouble(
1220                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1221             break;
1222         }
1223
1224         default:
1225             LOWERING_FAILED(m_node, "Bad use kind");
1226             break;
1227         }
1228     }
1229
1230     void compileArithDiv()
1231     {
1232         switch (m_node->binaryUseKind()) {
1233         case Int32Use: {
1234             LValue numerator = lowInt32(m_node->child1());
1235             LValue denominator = lowInt32(m_node->child2());
1236
1237             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
1238             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
1239             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
1240
1241             Vector<ValueFromBlock, 3> results;
1242
1243             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1244
1245             m_out.branch(
1246                 m_out.above(adjustedDenominator, m_out.int32One),
1247                 usually(continuation), rarely(unsafeDenominator));
1248
1249             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1250
1251             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1252
1253             if (shouldCheckOverflow(m_node->arithMode())) {
1254                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1255                 speculate(Overflow, noValue(), 0, cond);
1256                 m_out.jump(continuation);
1257             } else {
1258                 // This is the case where we convert the result to an int after we're done. So,
1259                 // if the denominator is zero, then the result should be zero.
1260                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1261                 // check above) and the numerator is -2^31 then the result should be -2^31.
1262
1263                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
1264                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
1265                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
1266
1267                 m_out.branch(
1268                     m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero));
1269
1270                 m_out.appendTo(divByZero, notDivByZero);
1271                 results.append(m_out.anchor(m_out.int32Zero));
1272                 m_out.jump(done);
1273
1274                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
1275                 m_out.branch(
1276                     m_out.equal(numerator, neg2ToThe31),
1277                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1278
1279                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1280                 results.append(m_out.anchor(neg2ToThe31));
1281                 m_out.jump(done);
1282             }
1283
1284             m_out.appendTo(continuation, done);
1285
1286             if (shouldCheckNegativeZero(m_node->arithMode())) {
1287                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
1288                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
1289
1290                 m_out.branch(
1291                     m_out.isZero32(numerator),
1292                     rarely(zeroNumerator), usually(numeratorContinuation));
1293
1294                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1295
1296                 speculate(
1297                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1298
1299                 m_out.jump(numeratorContinuation);
1300
1301                 m_out.appendTo(numeratorContinuation, innerLastNext);
1302             }
1303
1304             LValue result = m_out.div(numerator, denominator);
1305
1306             if (shouldCheckOverflow(m_node->arithMode())) {
1307                 speculate(
1308                     Overflow, noValue(), 0,
1309                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1310             }
1311
1312             results.append(m_out.anchor(result));
1313             m_out.jump(done);
1314
1315             m_out.appendTo(done, lastNext);
1316
1317             setInt32(m_out.phi(m_out.int32, results));
1318             break;
1319         }
1320
1321         case DoubleRepUse: {
1322             setDouble(m_out.doubleDiv(
1323                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1324             break;
1325         }
1326
1327         default:
1328             LOWERING_FAILED(m_node, "Bad use kind");
1329             break;
1330         }
1331     }
1332
1333     void compileArithMod()
1334     {
1335         switch (m_node->binaryUseKind()) {
1336         case Int32Use: {
1337             LValue numerator = lowInt32(m_node->child1());
1338             LValue denominator = lowInt32(m_node->child2());
1339
1340             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
1341             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
1342             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
1343
1344             Vector<ValueFromBlock, 3> results;
1345
1346             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1347
1348             m_out.branch(
1349                 m_out.above(adjustedDenominator, m_out.int32One),
1350                 usually(continuation), rarely(unsafeDenominator));
1351
1352             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1353
1354             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1355
1356             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
1357             // separate case for that. But it probably doesn't matter so much.
1358             if (shouldCheckOverflow(m_node->arithMode())) {
1359                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1360                 speculate(Overflow, noValue(), 0, cond);
1361                 m_out.jump(continuation);
1362             } else {
1363                 // This is the case where we convert the result to an int after we're done. So,
1364                 // if the denominator is zero, then the result should be result should be zero.
1365                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1366                 // check above) and the numerator is -2^31 then the result should be -2^31.
1367
1368                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
1369                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
1370                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
1371
1372                 m_out.branch(
1373                     m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero));
1374
1375                 m_out.appendTo(modByZero, notModByZero);
1376                 results.append(m_out.anchor(m_out.int32Zero));
1377                 m_out.jump(done);
1378
1379                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
1380                 m_out.branch(
1381                     m_out.equal(numerator, neg2ToThe31),
1382                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1383
1384                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1385                 results.append(m_out.anchor(m_out.int32Zero));
1386                 m_out.jump(done);
1387             }
1388
1389             m_out.appendTo(continuation, done);
1390
1391             LValue remainder = m_out.rem(numerator, denominator);
1392
1393             if (shouldCheckNegativeZero(m_node->arithMode())) {
1394                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
1395                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
1396
1397                 m_out.branch(
1398                     m_out.lessThan(numerator, m_out.int32Zero),
1399                     unsure(negativeNumerator), unsure(numeratorContinuation));
1400
1401                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1402
1403                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1404
1405                 m_out.jump(numeratorContinuation);
1406
1407                 m_out.appendTo(numeratorContinuation, innerLastNext);
1408             }
1409
1410             results.append(m_out.anchor(remainder));
1411             m_out.jump(done);
1412
1413             m_out.appendTo(done, lastNext);
1414
1415             setInt32(m_out.phi(m_out.int32, results));
1416             break;
1417         }
1418
1419         case DoubleRepUse: {
1420             setDouble(
1421                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1422             break;
1423         }
1424
1425         default:
1426             LOWERING_FAILED(m_node, "Bad use kind");
1427             break;
1428         }
1429     }
1430
1431     void compileArithMinOrMax()
1432     {
1433         switch (m_node->binaryUseKind()) {
1434         case Int32Use: {
1435             LValue left = lowInt32(m_node->child1());
1436             LValue right = lowInt32(m_node->child2());
1437
1438             setInt32(
1439                 m_out.select(
1440                     m_node->op() == ArithMin
1441                         ? m_out.lessThan(left, right)
1442                         : m_out.lessThan(right, left),
1443                     left, right));
1444             break;
1445         }
1446
1447         case DoubleRepUse: {
1448             LValue left = lowDouble(m_node->child1());
1449             LValue right = lowDouble(m_node->child2());
1450
1451             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1452             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1453
1454             Vector<ValueFromBlock, 2> results;
1455
1456             results.append(m_out.anchor(left));
1457             m_out.branch(
1458                 m_node->op() == ArithMin
1459                     ? m_out.doubleLessThan(left, right)
1460                     : m_out.doubleGreaterThan(left, right),
1461                 unsure(continuation), unsure(notLessThan));
1462
1463             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1464             results.append(m_out.anchor(m_out.select(
1465                 m_node->op() == ArithMin
1466                     ? m_out.doubleGreaterThanOrEqual(left, right)
1467                     : m_out.doubleLessThanOrEqual(left, right),
1468                 right, m_out.constDouble(PNaN))));
1469             m_out.jump(continuation);
1470
1471             m_out.appendTo(continuation, lastNext);
1472             setDouble(m_out.phi(m_out.doubleType, results));
1473             break;
1474         }
1475
1476         default:
1477             LOWERING_FAILED(m_node, "Bad use kind");
1478             break;
1479         }
1480     }
1481
1482     void compileArithAbs()
1483     {
1484         switch (m_node->child1().useKind()) {
1485         case Int32Use: {
1486             LValue value = lowInt32(m_node->child1());
1487
1488             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1489             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1490
1491             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1492
1493             setInt32(result);
1494             break;
1495         }
1496
1497         case DoubleRepUse: {
1498             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1499             break;
1500         }
1501
1502         default:
1503             LOWERING_FAILED(m_node, "Bad use kind");
1504             break;
1505         }
1506     }
1507
1508     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1509
1510     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1511
1512     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1513
1514     void compileArithFRound()
1515     {
1516         LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType);
1517         setDouble(m_out.fpCast(floatValue, m_out.doubleType));
1518     }
1519
1520     void compileArithNegate()
1521     {
1522         switch (m_node->child1().useKind()) {
1523         case Int32Use: {
1524             LValue value = lowInt32(m_node->child1());
1525
1526             LValue result;
1527             if (!shouldCheckOverflow(m_node->arithMode()))
1528                 result = m_out.neg(value);
1529             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
1530                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1531                 // does the trick, though.
1532                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1533                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1534                 result = m_out.extractValue(overflowResult, 0);
1535             } else {
1536                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1537                 result = m_out.neg(value);
1538             }
1539
1540             setInt32(result);
1541             break;
1542         }
1543
1544         case Int52RepUse: {
1545             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)) {
1546                 Int52Kind kind;
1547                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1548                 LValue result = m_out.neg(value);
1549                 if (shouldCheckNegativeZero(m_node->arithMode()))
1550                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1551                 setInt52(result, kind);
1552                 break;
1553             }
1554
1555             LValue value = lowInt52(m_node->child1());
1556             LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1557             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1558             LValue result = m_out.extractValue(overflowResult, 0);
1559             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1560             setInt52(result);
1561             break;
1562         }
1563
1564         case DoubleRepUse: {
1565             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1566             break;
1567         }
1568
1569         default:
1570             LOWERING_FAILED(m_node, "Bad use kind");
1571             break;
1572         }
1573     }
1574
1575     void compileBitAnd()
1576     {
1577         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1578     }
1579
1580     void compileBitOr()
1581     {
1582         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1583     }
1584
1585     void compileBitXor()
1586     {
1587         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1588     }
1589
1590     void compileBitRShift()
1591     {
1592         setInt32(m_out.aShr(
1593             lowInt32(m_node->child1()),
1594             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1595     }
1596
1597     void compileBitLShift()
1598     {
1599         setInt32(m_out.shl(
1600             lowInt32(m_node->child1()),
1601             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1602     }
1603
1604     void compileBitURShift()
1605     {
1606         setInt32(m_out.lShr(
1607             lowInt32(m_node->child1()),
1608             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1609     }
1610
1611     void compileUInt32ToNumber()
1612     {
1613         LValue value = lowInt32(m_node->child1());
1614
1615         if (doesOverflow(m_node->arithMode())) {
1616             setDouble(m_out.unsignedToDouble(value));
1617             return;
1618         }
1619
1620         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
1621         setInt32(value);
1622     }
1623
1624     void compileCheckStructure()
1625     {
1626         LValue cell = lowCell(m_node->child1());
1627
1628         ExitKind exitKind;
1629         if (m_node->child1()->op() == WeakJSConstant)
1630             exitKind = BadWeakConstantCache;
1631         else
1632             exitKind = BadCache;
1633
1634         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1635
1636         if (m_node->structureSet().size() == 1) {
1637             speculate(
1638                 exitKind, jsValueValue(cell), 0,
1639                 m_out.notEqual(structureID, weakStructure(m_node->structureSet()[0])));
1640             return;
1641         }
1642
1643         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1644
1645         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1646         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1647             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1648             m_out.branch(
1649                 m_out.equal(structureID, weakStructure(m_node->structureSet()[i])),
1650                 unsure(continuation), unsure(nextStructure));
1651             m_out.appendTo(nextStructure);
1652         }
1653
1654         speculate(
1655             exitKind, jsValueValue(cell), 0,
1656             m_out.notEqual(structureID, weakStructure(m_node->structureSet().last())));
1657
1658         m_out.jump(continuation);
1659         m_out.appendTo(continuation, lastNext);
1660     }
1661
1662     void compileStructureTransitionWatchpoint()
1663     {
1664         addWeakReference(m_node->structure());
1665         speculateCell(m_node->child1());
1666     }
1667
1668     void compileCheckFunction()
1669     {
1670         LValue cell = lowCell(m_node->child1());
1671
1672         speculate(
1673             BadFunction, jsValueValue(cell), m_node->child1().node(),
1674             m_out.notEqual(cell, weakPointer(m_node->function())));
1675     }
1676
1677     void compileCheckExecutable()
1678     {
1679         LValue cell = lowCell(m_node->child1());
1680
1681         speculate(
1682             BadExecutable, jsValueValue(cell), m_node->child1().node(),
1683             m_out.notEqual(
1684                 m_out.loadPtr(cell, m_heaps.JSFunction_executable),
1685                 weakPointer(m_node->executable())));
1686     }
1687
1688     void compileArrayifyToStructure()
1689     {
1690         LValue cell = lowCell(m_node->child1());
1691         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1692
1693         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1694         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1695
1696         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1697
1698         m_out.branch(
1699             m_out.notEqual(structureID, weakStructure(m_node->structure())),
1700             rarely(unexpectedStructure), usually(continuation));
1701
1702         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1703
1704         if (property) {
1705             switch (m_node->arrayMode().type()) {
1706             case Array::Int32:
1707             case Array::Double:
1708             case Array::Contiguous:
1709                 speculate(
1710                     Uncountable, noValue(), 0,
1711                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1712                 break;
1713             default:
1714                 break;
1715             }
1716         }
1717
1718         switch (m_node->arrayMode().type()) {
1719         case Array::Int32:
1720             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1721             break;
1722         case Array::Double:
1723             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1724             break;
1725         case Array::Contiguous:
1726             if (m_node->arrayMode().conversion() == Array::RageConvert)
1727                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1728             else
1729                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1730             break;
1731         case Array::ArrayStorage:
1732         case Array::SlowPutArrayStorage:
1733             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1734             break;
1735         default:
1736             LOWERING_FAILED(m_node, "Bad array type");
1737             return;
1738         }
1739
1740         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1741         speculate(
1742             BadIndexingType, jsValueValue(cell), 0,
1743             m_out.notEqual(structureID, weakStructure(m_node->structure())));
1744         m_out.jump(continuation);
1745
1746         m_out.appendTo(continuation, lastNext);
1747     }
1748
1749     void compilePutStructure()
1750     {
1751         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1752
1753         Structure* oldStructure = m_node->structureTransitionData().previousStructure;
1754         Structure* newStructure = m_node->structureTransitionData().newStructure;
1755         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
1756         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
1757         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
1758
1759         LValue cell = lowCell(m_node->child1());
1760         m_out.store32(
1761             weakStructure(newStructure),
1762             cell, m_heaps.JSCell_structureID);
1763     }
1764
1765     void compilePhantomPutStructure()
1766     {
1767         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1768     }
1769
1770     void compileGetById()
1771     {
1772         // Pretty much the only reason why we don't also support GetByIdFlush is because:
1773         // https://bugs.webkit.org/show_bug.cgi?id=125711
1774
1775         switch (m_node->child1().useKind()) {
1776         case CellUse: {
1777             setJSValue(getById(lowCell(m_node->child1())));
1778             return;
1779         }
1780
1781         case UntypedUse: {
1782             // This is pretty weird, since we duplicate the slow path both here and in the
1783             // code generated by the IC. We should investigate making this less bad.
1784             // https://bugs.webkit.org/show_bug.cgi?id=127830
1785             LValue value = lowJSValue(m_node->child1());
1786
1787             LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case"));
1788             LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case"));
1789             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation"));
1790
1791             m_out.branch(isCell(value), unsure(cellCase), unsure(notCellCase));
1792
1793             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
1794             ValueFromBlock cellResult = m_out.anchor(getById(value));
1795             m_out.jump(continuation);
1796
1797             m_out.appendTo(notCellCase, continuation);
1798             ValueFromBlock notCellResult = m_out.anchor(vmCall(
1799                 m_out.operation(operationGetById),
1800                 m_callFrame, getUndef(m_out.intPtr), value,
1801                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
1802             m_out.jump(continuation);
1803
1804             m_out.appendTo(continuation, lastNext);
1805             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
1806             return;
1807         }
1808
1809         default:
1810             LOWERING_FAILED(m_node, "Bad use kind");
1811             return;
1812         }
1813     }
1814
1815     void compilePutById()
1816     {
1817         // See above; CellUse is easier so we do only that for now.
1818         ASSERT(m_node->child1().useKind() == CellUse);
1819
1820         LValue base = lowCell(m_node->child1());
1821         LValue value = lowJSValue(m_node->child2());
1822         StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
1823
1824         // Arguments: id, bytes, target, numArgs, args...
1825         unsigned stackmapID = m_stackmapIDs++;
1826
1827         if (Options::verboseCompilation())
1828             dataLog("    Emitting PutById patchpoint with stackmap #", stackmapID, "\n");
1829
1830         LValue call = m_out.call(
1831             m_out.patchpointVoidIntrinsic(),
1832             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()),
1833             constNull(m_out.ref8), m_out.constInt32(2), base, value);
1834         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
1835
1836         m_ftlState.putByIds.append(PutByIdDescriptor(
1837             stackmapID, m_node->origin.semantic, uid,
1838             m_graph.executableFor(m_node->origin.semantic)->ecmaMode(),
1839             m_node->op() == PutByIdDirect ? Direct : NotDirect));
1840     }
1841
1842     void compileGetButterfly()
1843     {
1844         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1845     }
1846
1847     void compileConstantStoragePointer()
1848     {
1849         setStorage(m_out.constIntPtr(m_node->storagePointer()));
1850     }
1851
1852     void compileGetIndexedPropertyStorage()
1853     {
1854         LValue cell = lowCell(m_node->child1());
1855
1856         if (m_node->arrayMode().type() == Array::String) {
1857             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String slow case"));
1858             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String continuation"));
1859
1860             ValueFromBlock fastResult = m_out.anchor(
1861                 m_out.loadPtr(cell, m_heaps.JSString_value));
1862
1863             m_out.branch(
1864                 m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath));
1865
1866             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
1867
1868             ValueFromBlock slowResult = m_out.anchor(
1869                 vmCall(m_out.operation(operationResolveRope), m_callFrame, cell));
1870
1871             m_out.jump(continuation);
1872
1873             m_out.appendTo(continuation, lastNext);
1874
1875             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
1876             return;
1877         }
1878
1879         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
1880     }
1881
1882     void compileCheckArray()
1883     {
1884         Edge edge = m_node->child1();
1885         LValue cell = lowCell(edge);
1886
1887         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, m_state.forNode(edge)))
1888             return;
1889
1890         speculate(
1891             BadIndexingType, jsValueValue(cell), 0,
1892             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
1893     }
1894
1895     void compileGetTypedArrayByteOffset()
1896     {
1897         LValue basePtr = lowCell(m_node->child1());
1898
1899         LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array"));
1900         LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array"));
1901         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch"));
1902
1903         LValue baseAddress = m_out.addPtr(basePtr, JSArrayBufferView::offsetOfMode());
1904         m_out.branch(
1905             m_out.notEqual(baseAddress , m_out.constIntPtr(WastefulTypedArray)),
1906             unsure(simpleCase), unsure(wastefulCase));
1907
1908         // begin simple case
1909         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
1910
1911         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
1912
1913         m_out.jump(continuation);
1914
1915         // begin wasteful case
1916         m_out.appendTo(wastefulCase, continuation);
1917
1918         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
1919         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
1920         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
1921         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
1922
1923         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(dataPtr, vectorPtr));
1924
1925         m_out.jump(continuation);
1926         m_out.appendTo(continuation, lastNext);
1927
1928         // output
1929         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
1930     }
1931
1932     void compileGetMyArgumentsLength()
1933     {
1934         checkArgumentsNotCreated();
1935
1936         RELEASE_ASSERT(!m_node->origin.semantic.inlineCallFrame);
1937         setInt32(m_out.add(m_out.load32NonNegative(payloadFor(JSStack::ArgumentCount)), m_out.constInt32(-1)));
1938     }
1939
1940     void compileGetMyArgumentByVal()
1941     {
1942         checkArgumentsNotCreated();
1943
1944         CodeOrigin codeOrigin = m_node->origin.semantic;
1945
1946         LValue zeroBasedIndex = lowInt32(m_node->child1());
1947         LValue oneBasedIndex = m_out.add(zeroBasedIndex, m_out.int32One);
1948
1949         LValue limit;
1950         if (codeOrigin.inlineCallFrame)
1951             limit = m_out.constInt32(codeOrigin.inlineCallFrame->arguments.size());
1952         else
1953             limit = m_out.load32(payloadFor(JSStack::ArgumentCount));
1954
1955         speculate(Uncountable, noValue(), 0, m_out.aboveOrEqual(oneBasedIndex, limit));
1956
1957         SymbolTable* symbolTable = m_graph.baselineCodeBlockFor(codeOrigin)->symbolTable();
1958         if (symbolTable->slowArguments()) {
1959             // FIXME: FTL should support activations.
1960             // https://bugs.webkit.org/show_bug.cgi?id=129576
1961
1962             LOWERING_FAILED(m_node, "Unimplemented");
1963             return;
1964         }
1965
1966         TypedPointer base;
1967         if (codeOrigin.inlineCallFrame) {
1968             VirtualRegister reg;
1969             if (codeOrigin.inlineCallFrame->arguments.size() <= 1)
1970                 reg = virtualRegisterForLocal(0); // Doesn't matter what we do since we would have exited anyway.
1971             else
1972                 reg = codeOrigin.inlineCallFrame->arguments[1].virtualRegister();
1973             base = addressFor(reg);
1974         } else
1975             base = addressFor(virtualRegisterForArgument(1));
1976
1977         LValue pointer = m_out.baseIndex(
1978             base.value(), m_out.zeroExt(zeroBasedIndex, m_out.intPtr), ScaleEight);
1979         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
1980     }
1981
1982     void compileGetArrayLength()
1983     {
1984         switch (m_node->arrayMode().type()) {
1985         case Array::Int32:
1986         case Array::Double:
1987         case Array::Contiguous: {
1988             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1989             return;
1990         }
1991
1992         case Array::String: {
1993             LValue string = lowCell(m_node->child1());
1994             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
1995             return;
1996         }
1997
1998         default:
1999             if (isTypedView(m_node->arrayMode().typedArrayType())) {
2000                 setInt32(
2001                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2002                 return;
2003             }
2004
2005             LOWERING_FAILED(m_node, "Bad array type");
2006             return;
2007         }
2008     }
2009
2010     void compileCheckInBounds()
2011     {
2012         speculate(
2013             OutOfBounds, noValue(), 0,
2014             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2015     }
2016
2017     void compileGetByVal()
2018     {
2019         switch (m_node->arrayMode().type()) {
2020         case Array::Int32:
2021         case Array::Contiguous: {
2022             LValue index = lowInt32(m_node->child2());
2023             LValue storage = lowStorage(m_node->child3());
2024
2025             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2026                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2027
2028             if (m_node->arrayMode().isInBounds()) {
2029                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2030                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
2031                 setJSValue(result);
2032                 return;
2033             }
2034
2035             LValue base = lowCell(m_node->child1());
2036
2037             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous fast case"));
2038             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous slow case"));
2039             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous continuation"));
2040
2041             m_out.branch(
2042                 m_out.aboveOrEqual(
2043                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2044                 rarely(slowCase), usually(fastCase));
2045
2046             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2047
2048             ValueFromBlock fastResult = m_out.anchor(
2049                 m_out.load64(baseIndex(heap, storage, index, m_node->child2())));
2050             m_out.branch(
2051                 m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation));
2052
2053             m_out.appendTo(slowCase, continuation);
2054             ValueFromBlock slowResult = m_out.anchor(
2055                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2056             m_out.jump(continuation);
2057
2058             m_out.appendTo(continuation, lastNext);
2059             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2060             return;
2061         }
2062
2063         case Array::Double: {
2064             LValue index = lowInt32(m_node->child2());
2065             LValue storage = lowStorage(m_node->child3());
2066
2067             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2068
2069             if (m_node->arrayMode().isInBounds()) {
2070                 LValue result = m_out.loadDouble(
2071                     baseIndex(heap, storage, index, m_node->child2()));
2072
2073                 if (!m_node->arrayMode().isSaneChain()) {
2074                     speculate(
2075                         LoadFromHole, noValue(), 0,
2076                         m_out.doubleNotEqualOrUnordered(result, result));
2077                 }
2078                 setDouble(result);
2079                 break;
2080             }
2081
2082             LValue base = lowCell(m_node->child1());
2083
2084             LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetByVal double in bounds"));
2085             LBasicBlock boxPath = FTL_NEW_BLOCK(m_out, ("GetByVal double boxing"));
2086             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal double slow case"));
2087             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal double continuation"));
2088
2089             m_out.branch(
2090                 m_out.aboveOrEqual(
2091                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2092                 rarely(slowCase), usually(inBounds));
2093
2094             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2095             LValue doubleValue = m_out.loadDouble(
2096                 baseIndex(heap, storage, index, m_node->child2()));
2097             m_out.branch(
2098                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2099                 rarely(slowCase), usually(boxPath));
2100
2101             m_out.appendTo(boxPath, slowCase);
2102             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2103             m_out.jump(continuation);
2104
2105             m_out.appendTo(slowCase, continuation);
2106             ValueFromBlock slowResult = m_out.anchor(
2107                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2108             m_out.jump(continuation);
2109
2110             m_out.appendTo(continuation, lastNext);
2111             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2112             return;
2113         }
2114
2115         case Array::Generic: {
2116             setJSValue(vmCall(
2117                 m_out.operation(operationGetByVal), m_callFrame,
2118                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2119             return;
2120         }
2121
2122         case Array::String: {
2123             compileStringCharAt();
2124             return;
2125         }
2126
2127         default: {
2128             LValue index = lowInt32(m_node->child2());
2129             LValue storage = lowStorage(m_node->child3());
2130
2131             TypedArrayType type = m_node->arrayMode().typedArrayType();
2132
2133             if (isTypedView(type)) {
2134                 TypedPointer pointer = TypedPointer(
2135                     m_heaps.typedArrayProperties,
2136                     m_out.add(
2137                         storage,
2138                         m_out.shl(
2139                             m_out.zeroExt(index, m_out.intPtr),
2140                             m_out.constIntPtr(logElementSize(type)))));
2141
2142                 if (isInt(type)) {
2143                     LValue result;
2144                     switch (elementSize(type)) {
2145                     case 1:
2146                         result = m_out.load8(pointer);
2147                         break;
2148                     case 2:
2149                         result = m_out.load16(pointer);
2150                         break;
2151                     case 4:
2152                         result = m_out.load32(pointer);
2153                         break;
2154                     default:
2155                         LOWERING_FAILED(m_node, "Bad element size");
2156                         return;
2157                     }
2158
2159                     if (elementSize(type) < 4) {
2160                         if (isSigned(type))
2161                             result = m_out.signExt(result, m_out.int32);
2162                         else
2163                             result = m_out.zeroExt(result, m_out.int32);
2164                         setInt32(result);
2165                         return;
2166                     }
2167
2168                     if (isSigned(type)) {
2169                         setInt32(result);
2170                         return;
2171                     }
2172
2173                     if (m_node->shouldSpeculateInt32()) {
2174                         speculate(
2175                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2176                         setInt32(result);
2177                         return;
2178                     }
2179
2180                     if (m_node->shouldSpeculateMachineInt()) {
2181                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2182                         return;
2183                     }
2184
2185                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
2186                     return;
2187                 }
2188
2189                 ASSERT(isFloat(type));
2190
2191                 LValue result;
2192                 switch (type) {
2193                 case TypeFloat32:
2194                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
2195                     break;
2196                 case TypeFloat64:
2197                     result = m_out.loadDouble(pointer);
2198                     break;
2199                 default:
2200                     LOWERING_FAILED(m_node, "Bad typed array type");
2201                     return;
2202                 }
2203
2204                 setDouble(result);
2205                 return;
2206             }
2207
2208             LOWERING_FAILED(m_node, "Bad array type");
2209             return;
2210         } }
2211     }
2212
2213     void compilePutByVal()
2214     {
2215         Edge child1 = m_graph.varArgChild(m_node, 0);
2216         Edge child2 = m_graph.varArgChild(m_node, 1);
2217         Edge child3 = m_graph.varArgChild(m_node, 2);
2218         Edge child4 = m_graph.varArgChild(m_node, 3);
2219         Edge child5 = m_graph.varArgChild(m_node, 4);
2220
2221         switch (m_node->arrayMode().type()) {
2222         case Array::Generic: {
2223             V_JITOperation_EJJJ operation;
2224             if (m_node->op() == PutByValDirect) {
2225                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2226                     operation = operationPutByValDirectStrict;
2227                 else
2228                     operation = operationPutByValDirectNonStrict;
2229             } else {
2230                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2231                     operation = operationPutByValStrict;
2232                 else
2233                     operation = operationPutByValNonStrict;
2234             }
2235
2236             vmCall(
2237                 m_out.operation(operation), m_callFrame,
2238                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2239             return;
2240         }
2241
2242         default:
2243             break;
2244         }
2245
2246         LValue base = lowCell(child1);
2247         LValue index = lowInt32(child2);
2248         LValue storage = lowStorage(child4);
2249
2250         switch (m_node->arrayMode().type()) {
2251         case Array::Int32:
2252         case Array::Double:
2253         case Array::Contiguous: {
2254             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
2255             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2256
2257             switch (m_node->arrayMode().type()) {
2258             case Array::Int32:
2259             case Array::Contiguous: {
2260                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2261
2262                 if (m_node->arrayMode().type() == Array::Int32)
2263                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2264
2265                 TypedPointer elementPointer = m_out.baseIndex(
2266                     m_node->arrayMode().type() == Array::Int32 ?
2267                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2268                     storage, m_out.zeroExt(index, m_out.intPtr),
2269                     m_state.forNode(child2).m_value);
2270
2271                 if (m_node->op() == PutByValAlias) {
2272                     m_out.store64(value, elementPointer);
2273                     break;
2274                 }
2275
2276                 contiguousPutByValOutOfBounds(
2277                     codeBlock()->isStrictMode()
2278                     ? operationPutByValBeyondArrayBoundsStrict
2279                     : operationPutByValBeyondArrayBoundsNonStrict,
2280                     base, storage, index, value, continuation);
2281
2282                 m_out.store64(value, elementPointer);
2283                 break;
2284             }
2285
2286             case Array::Double: {
2287                 LValue value = lowDouble(child3);
2288
2289                 FTL_TYPE_CHECK(
2290                     doubleValue(value), child3, SpecDoubleReal,
2291                     m_out.doubleNotEqualOrUnordered(value, value));
2292
2293                 TypedPointer elementPointer = m_out.baseIndex(
2294                     m_heaps.indexedDoubleProperties,
2295                     storage, m_out.zeroExt(index, m_out.intPtr),
2296                     m_state.forNode(child2).m_value);
2297
2298                 if (m_node->op() == PutByValAlias) {
2299                     m_out.storeDouble(value, elementPointer);
2300                     break;
2301                 }
2302
2303                 contiguousPutByValOutOfBounds(
2304                     codeBlock()->isStrictMode()
2305                     ? operationPutDoubleByValBeyondArrayBoundsStrict
2306                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2307                     base, storage, index, value, continuation);
2308
2309                 m_out.storeDouble(value, elementPointer);
2310                 break;
2311             }
2312
2313             default:
2314                 LOWERING_FAILED(m_node, "Bad array type");
2315                 return;
2316             }
2317
2318             m_out.jump(continuation);
2319             m_out.appendTo(continuation, outerLastNext);
2320             return;
2321         }
2322
2323         default:
2324             TypedArrayType type = m_node->arrayMode().typedArrayType();
2325
2326             if (isTypedView(type)) {
2327                 TypedPointer pointer = TypedPointer(
2328                     m_heaps.typedArrayProperties,
2329                     m_out.add(
2330                         storage,
2331                         m_out.shl(
2332                             m_out.zeroExt(index, m_out.intPtr),
2333                             m_out.constIntPtr(logElementSize(type)))));
2334
2335                 LType refType;
2336                 LValue valueToStore;
2337
2338                 if (isInt(type)) {
2339                     LValue intValue;
2340                     switch (child3.useKind()) {
2341                     case Int52RepUse:
2342                     case Int32Use: {
2343                         if (child3.useKind() == Int32Use)
2344                             intValue = lowInt32(child3);
2345                         else
2346                             intValue = m_out.castToInt32(lowStrictInt52(child3));
2347
2348                         if (isClamped(type)) {
2349                             ASSERT(elementSize(type) == 1);
2350
2351                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
2352                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
2353
2354                             Vector<ValueFromBlock, 2> intValues;
2355                             intValues.append(m_out.anchor(m_out.int32Zero));
2356                             m_out.branch(
2357                                 m_out.lessThan(intValue, m_out.int32Zero),
2358                                 unsure(continuation), unsure(atLeastZero));
2359
2360                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
2361
2362                             intValues.append(m_out.anchor(m_out.select(
2363                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
2364                                 m_out.constInt32(255),
2365                                 intValue)));
2366                             m_out.jump(continuation);
2367
2368                             m_out.appendTo(continuation, lastNext);
2369                             intValue = m_out.phi(m_out.int32, intValues);
2370                         }
2371                         break;
2372                     }
2373
2374                     case DoubleRepUse: {
2375                         LValue doubleValue = lowDouble(child3);
2376
2377                         if (isClamped(type)) {
2378                             ASSERT(elementSize(type) == 1);
2379
2380                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
2381                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
2382                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
2383
2384                             Vector<ValueFromBlock, 3> intValues;
2385                             intValues.append(m_out.anchor(m_out.int32Zero));
2386                             m_out.branch(
2387                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
2388                                 unsure(continuation), unsure(atLeastZero));
2389
2390                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
2391                             intValues.append(m_out.anchor(m_out.constInt32(255)));
2392                             m_out.branch(
2393                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
2394                                 unsure(continuation), unsure(withinRange));
2395
2396                             m_out.appendTo(withinRange, continuation);
2397                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
2398                             m_out.jump(continuation);
2399
2400                             m_out.appendTo(continuation, lastNext);
2401                             intValue = m_out.phi(m_out.int32, intValues);
2402                         } else
2403                             intValue = doubleToInt32(doubleValue);
2404                         break;
2405                     }
2406
2407                     default:
2408                         LOWERING_FAILED(m_node, "Bad use kind");
2409                         return;
2410                     }
2411
2412                     switch (elementSize(type)) {
2413                     case 1:
2414                         valueToStore = m_out.intCast(intValue, m_out.int8);
2415                         refType = m_out.ref8;
2416                         break;
2417                     case 2:
2418                         valueToStore = m_out.intCast(intValue, m_out.int16);
2419                         refType = m_out.ref16;
2420                         break;
2421                     case 4:
2422                         valueToStore = intValue;
2423                         refType = m_out.ref32;
2424                         break;
2425                     default:
2426                         LOWERING_FAILED(m_node, "Bad element size");
2427                         return;
2428                     }
2429                 } else /* !isInt(type) */ {
2430                     LValue value = lowDouble(child3);
2431                     switch (type) {
2432                     case TypeFloat32:
2433                         valueToStore = m_out.fpCast(value, m_out.floatType);
2434                         refType = m_out.refFloat;
2435                         break;
2436                     case TypeFloat64:
2437                         valueToStore = value;
2438                         refType = m_out.refDouble;
2439                         break;
2440                     default:
2441                         LOWERING_FAILED(m_node, "Bad typed array type");
2442                         return;
2443                     }
2444                 }
2445
2446                 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
2447                     m_out.store(valueToStore, pointer, refType);
2448                 else {
2449                     LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case"));
2450                     LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation"));
2451
2452                     m_out.branch(
2453                         m_out.aboveOrEqual(index, lowInt32(child5)),
2454                         unsure(continuation), unsure(isInBounds));
2455
2456                     LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
2457                     m_out.store(valueToStore, pointer, refType);
2458                     m_out.jump(continuation);
2459
2460                     m_out.appendTo(continuation, lastNext);
2461                 }
2462
2463                 return;
2464             }
2465
2466             LOWERING_FAILED(m_node, "Bad array type");
2467             return;
2468         }
2469     }
2470
2471     void compileArrayPush()
2472     {
2473         LValue base = lowCell(m_node->child1());
2474         LValue storage = lowStorage(m_node->child3());
2475
2476         switch (m_node->arrayMode().type()) {
2477         case Array::Int32:
2478         case Array::Contiguous:
2479         case Array::Double: {
2480             LValue value;
2481             LType refType;
2482
2483             if (m_node->arrayMode().type() != Array::Double) {
2484                 value = lowJSValue(m_node->child2(), ManualOperandSpeculation);
2485                 if (m_node->arrayMode().type() == Array::Int32) {
2486                     FTL_TYPE_CHECK(
2487                         jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value));
2488                 }
2489                 refType = m_out.ref64;
2490             } else {
2491                 value = lowDouble(m_node->child2());
2492                 FTL_TYPE_CHECK(
2493                     doubleValue(value), m_node->child2(), SpecDoubleReal,
2494                     m_out.doubleNotEqualOrUnordered(value, value));
2495                 refType = m_out.refDouble;
2496             }
2497
2498             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2499
2500             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2501
2502             LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path"));
2503             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path"));
2504             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation"));
2505
2506             m_out.branch(
2507                 m_out.aboveOrEqual(
2508                     prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
2509                 rarely(slowPath), usually(fastPath));
2510
2511             LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
2512             m_out.store(
2513                 value,
2514                 m_out.baseIndex(heap, storage, m_out.zeroExt(prevLength, m_out.intPtr)),
2515                 refType);
2516             LValue newLength = m_out.add(prevLength, m_out.int32One);
2517             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2518
2519             ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
2520             m_out.jump(continuation);
2521
2522             m_out.appendTo(slowPath, continuation);
2523             LValue operation;
2524             if (m_node->arrayMode().type() != Array::Double)
2525                 operation = m_out.operation(operationArrayPush);
2526             else
2527                 operation = m_out.operation(operationArrayPushDouble);
2528             ValueFromBlock slowResult = m_out.anchor(
2529                 vmCall(operation, m_callFrame, value, base));
2530             m_out.jump(continuation);
2531
2532             m_out.appendTo(continuation, lastNext);
2533             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2534             return;
2535         }
2536
2537         default:
2538             LOWERING_FAILED(m_node, "Bad array type");
2539             return;
2540         }
2541     }
2542
2543     void compileArrayPop()
2544     {
2545         LValue base = lowCell(m_node->child1());
2546         LValue storage = lowStorage(m_node->child2());
2547
2548         switch (m_node->arrayMode().type()) {
2549         case Array::Int32:
2550         case Array::Double:
2551         case Array::Contiguous: {
2552             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2553
2554             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case"));
2555             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case"));
2556             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation"));
2557
2558             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2559
2560             Vector<ValueFromBlock, 3> results;
2561             results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
2562             m_out.branch(
2563                 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
2564
2565             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2566             LValue newLength = m_out.sub(prevLength, m_out.int32One);
2567             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2568             TypedPointer pointer = m_out.baseIndex(
2569                 heap, storage, m_out.zeroExt(newLength, m_out.intPtr));
2570             if (m_node->arrayMode().type() != Array::Double) {
2571                 LValue result = m_out.load64(pointer);
2572                 m_out.store64(m_out.int64Zero, pointer);
2573                 results.append(m_out.anchor(result));
2574                 m_out.branch(
2575                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
2576             } else {
2577                 LValue result = m_out.loadDouble(pointer);
2578                 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
2579                 results.append(m_out.anchor(boxDouble(result)));
2580                 m_out.branch(
2581                     m_out.doubleEqual(result, result),
2582                     usually(continuation), rarely(slowCase));
2583             }
2584
2585             m_out.appendTo(slowCase, continuation);
2586             results.append(m_out.anchor(vmCall(
2587                 m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base)));
2588             m_out.jump(continuation);
2589
2590             m_out.appendTo(continuation, lastNext);
2591             setJSValue(m_out.phi(m_out.int64, results));
2592             return;
2593         }
2594
2595         default:
2596             LOWERING_FAILED(m_node, "Bad array type");
2597             return;
2598         }
2599     }
2600
2601     void compileNewObject()
2602     {
2603         Structure* structure = m_node->structure();
2604         size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
2605         MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize);
2606
2607         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewObject slow path"));
2608         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewObject continuation"));
2609
2610         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2611
2612         ValueFromBlock fastResult = m_out.anchor(allocateObject(
2613             m_out.constIntPtr(allocator), structure, m_out.intPtrZero, slowPath));
2614
2615         m_out.jump(continuation);
2616
2617         m_out.appendTo(slowPath, continuation);
2618
2619         ValueFromBlock slowResult = m_out.anchor(vmCall(
2620             m_out.operation(operationNewObject), m_callFrame, m_out.constIntPtr(structure)));
2621         m_out.jump(continuation);
2622
2623         m_out.appendTo(continuation, lastNext);
2624         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2625     }
2626
2627     void compileNewArray()
2628     {
2629         // First speculate appropriately on all of the children. Do this unconditionally up here
2630         // because some of the slow paths may otherwise forget to do it. It's sort of arguable
2631         // that doing the speculations up here might be unprofitable for RA - so we can consider
2632         // sinking this to below the allocation fast path if we find that this has a lot of
2633         // register pressure.
2634         for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex)
2635             speculate(m_graph.varArgChild(m_node, operandIndex));
2636
2637         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2638         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
2639             m_node->indexingType());
2640
2641         RELEASE_ASSERT(structure->indexingType() == m_node->indexingType());
2642
2643         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
2644             unsigned numElements = m_node->numChildren();
2645
2646             ArrayValues arrayValues = allocateJSArray(structure, numElements);
2647
2648             for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
2649                 Edge edge = m_graph.varArgChild(m_node, operandIndex);
2650
2651                 switch (m_node->indexingType()) {
2652                 case ALL_BLANK_INDEXING_TYPES:
2653                 case ALL_UNDECIDED_INDEXING_TYPES:
2654                     CRASH();
2655                     break;
2656
2657                 case ALL_DOUBLE_INDEXING_TYPES:
2658                     m_out.storeDouble(
2659                         lowDouble(edge),
2660                         arrayValues.butterfly, m_heaps.indexedDoubleProperties[operandIndex]);
2661                     break;
2662
2663                 case ALL_INT32_INDEXING_TYPES:
2664                 case ALL_CONTIGUOUS_INDEXING_TYPES:
2665                     m_out.store64(
2666                         lowJSValue(edge, ManualOperandSpeculation),
2667                         arrayValues.butterfly,
2668                         m_heaps.forIndexingType(m_node->indexingType())->at(operandIndex));
2669                     break;
2670
2671                 default:
2672                     CRASH();
2673                 }
2674             }
2675
2676             setJSValue(arrayValues.array);
2677             return;
2678         }
2679
2680         if (!m_node->numChildren()) {
2681             setJSValue(vmCall(
2682                 m_out.operation(operationNewEmptyArray), m_callFrame,
2683                 m_out.constIntPtr(structure)));
2684             return;
2685         }
2686
2687         size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren();
2688         ASSERT(scratchSize);
2689         ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
2690         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
2691
2692         for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
2693             Edge edge = m_graph.varArgChild(m_node, operandIndex);
2694             m_out.store64(
2695                 lowJSValue(edge, ManualOperandSpeculation),
2696                 m_out.absolute(buffer + operandIndex));
2697         }
2698
2699         m_out.storePtr(
2700             m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->activeLengthPtr()));
2701
2702         LValue result = vmCall(
2703             m_out.operation(operationNewArray), m_callFrame,
2704             m_out.constIntPtr(structure), m_out.constIntPtr(buffer),
2705             m_out.constIntPtr(m_node->numChildren()));
2706
2707         m_out.storePtr(m_out.intPtrZero, m_out.absolute(scratchBuffer->activeLengthPtr()));
2708
2709         setJSValue(result);
2710     }
2711
2712     void compileNewArrayBuffer()
2713     {
2714         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2715         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
2716             m_node->indexingType());
2717
2718         RELEASE_ASSERT(structure->indexingType() == m_node->indexingType());
2719
2720         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
2721             unsigned numElements = m_node->numConstants();
2722
2723             ArrayValues arrayValues = allocateJSArray(structure, numElements);
2724
2725             JSValue* data = codeBlock()->constantBuffer(m_node->startConstant());
2726             for (unsigned index = 0; index < m_node->numConstants(); ++index) {
2727                 int64_t value;
2728                 if (hasDouble(m_node->indexingType()))
2729                     value = bitwise_cast<int64_t>(data[index].asNumber());
2730                 else
2731                     value = JSValue::encode(data[index]);
2732
2733                 m_out.store64(
2734                     m_out.constInt64(value),
2735                     arrayValues.butterfly,
2736                     m_heaps.forIndexingType(m_node->indexingType())->at(index));
2737             }
2738
2739             setJSValue(arrayValues.array);
2740             return;
2741         }
2742
2743         setJSValue(vmCall(
2744             m_out.operation(operationNewArrayBuffer), m_callFrame,
2745             m_out.constIntPtr(structure), m_out.constIntPtr(m_node->startConstant()),
2746             m_out.constIntPtr(m_node->numConstants())));
2747     }
2748
2749     void compileNewArrayWithSize()
2750     {
2751         LValue publicLength = lowInt32(m_node->child1());
2752
2753         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2754         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
2755             m_node->indexingType());
2756
2757         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
2758             ASSERT(
2759                 hasUndecided(structure->indexingType())
2760                 || hasInt32(structure->indexingType())
2761                 || hasDouble(structure->indexingType())
2762                 || hasContiguous(structure->indexingType()));
2763
2764             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fast case"));
2765             LBasicBlock largeCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize large case"));
2766             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fail case"));
2767             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize slow case"));
2768             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize continuation"));
2769
2770             m_out.branch(
2771                 m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)),
2772                 rarely(largeCase), usually(fastCase));
2773
2774             LBasicBlock lastNext = m_out.appendTo(fastCase, largeCase);
2775
2776             // We don't round up to BASE_VECTOR_LEN for new Array(blah).
2777             LValue vectorLength = publicLength;
2778
2779             LValue payloadSize =
2780                 m_out.shl(m_out.zeroExt(vectorLength, m_out.intPtr), m_out.constIntPtr(3));
2781
2782             LValue butterflySize = m_out.add(
2783                 payloadSize, m_out.constIntPtr(sizeof(IndexingHeader)));
2784
2785             LValue endOfStorage = allocateBasicStorageAndGetEnd(butterflySize, failCase);
2786
2787             LValue butterfly = m_out.sub(endOfStorage, payloadSize);
2788
2789             LValue object = allocateObject<JSArray>(
2790                 structure, butterfly, failCase);
2791
2792             m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength);
2793             m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength);
2794
2795             if (hasDouble(m_node->indexingType())) {
2796                 LBasicBlock initLoop = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init loop"));
2797                 LBasicBlock initDone = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init done"));
2798
2799                 ValueFromBlock originalIndex = m_out.anchor(vectorLength);
2800                 ValueFromBlock originalPointer = m_out.anchor(butterfly);
2801                 m_out.branch(
2802                     m_out.notZero32(vectorLength), unsure(initLoop), unsure(initDone));
2803
2804                 LBasicBlock initLastNext = m_out.appendTo(initLoop, initDone);
2805                 LValue index = m_out.phi(m_out.int32, originalIndex);
2806                 LValue pointer = m_out.phi(m_out.intPtr, originalPointer);
2807
2808                 m_out.store64(
2809                     m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
2810                     TypedPointer(m_heaps.indexedDoubleProperties.atAnyIndex(), pointer));
2811
2812                 LValue nextIndex = m_out.sub(index, m_out.int32One);
2813                 addIncoming(index, m_out.anchor(nextIndex));
2814                 addIncoming(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight)));
2815                 m_out.branch(
2816                     m_out.notZero32(nextIndex), unsure(initLoop), unsure(initDone));
2817
2818                 m_out.appendTo(initDone, initLastNext);
2819             }
2820
2821             ValueFromBlock fastResult = m_out.anchor(object);
2822             m_out.jump(continuation);
2823
2824             m_out.appendTo(largeCase, failCase);
2825             ValueFromBlock largeStructure = m_out.anchor(m_out.constIntPtr(
2826                 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)));
2827             m_out.jump(slowCase);
2828
2829             m_out.appendTo(failCase, slowCase);
2830             ValueFromBlock failStructure = m_out.anchor(m_out.constIntPtr(structure));
2831             m_out.jump(slowCase);
2832
2833             m_out.appendTo(slowCase, continuation);
2834             LValue structureValue = m_out.phi(
2835                 m_out.intPtr, largeStructure, failStructure);
2836             ValueFromBlock slowResult = m_out.anchor(vmCall(
2837                 m_out.operation(operationNewArrayWithSize),
2838                 m_callFrame, structureValue, publicLength));
2839             m_out.jump(continuation);
2840
2841             m_out.appendTo(continuation, lastNext);
2842             setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2843             return;
2844         }
2845
2846         LValue structureValue = m_out.select(
2847             m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)),
2848             m_out.constIntPtr(
2849                 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)),
2850             m_out.constIntPtr(structure));
2851         setJSValue(vmCall(m_out.operation(operationNewArrayWithSize), m_callFrame, structureValue, publicLength));
2852     }
2853
2854     void compileAllocatePropertyStorage()
2855     {
2856         StructureTransitionData& data = m_node->structureTransitionData();
2857         LValue object = lowCell(m_node->child1());
2858
2859         setStorage(allocatePropertyStorage(object, data.previousStructure));
2860     }
2861
2862     void compileReallocatePropertyStorage()
2863     {
2864         StructureTransitionData& data = m_node->structureTransitionData();
2865         LValue object = lowCell(m_node->child1());
2866         LValue oldStorage = lowStorage(m_node->child2());
2867
2868         setStorage(
2869             reallocatePropertyStorage(
2870                 object, oldStorage, data.previousStructure, data.newStructure));
2871     }
2872
2873     void compileToString()
2874     {
2875         switch (m_node->child1().useKind()) {
2876         case StringObjectUse: {
2877             LValue cell = lowCell(m_node->child1());
2878             speculateStringObjectForCell(m_node->child1(), cell);
2879             m_interpreter.filter(m_node->child1(), SpecStringObject);
2880
2881             setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
2882             return;
2883         }
2884
2885         case StringOrStringObjectUse: {
2886             LValue cell = lowCell(m_node->child1());
2887             LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2888
2889             LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject not string case"));
2890             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject continuation"));
2891
2892             ValueFromBlock simpleResult = m_out.anchor(cell);
2893             m_out.branch(
2894                 m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
2895                 unsure(continuation), unsure(notString));
2896
2897             LBasicBlock lastNext = m_out.appendTo(notString, continuation);
2898             speculateStringObjectForStructureID(m_node->child1(), structureID);
2899             ValueFromBlock unboxedResult = m_out.anchor(
2900                 m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
2901             m_out.jump(continuation);
2902
2903             m_out.appendTo(continuation, lastNext);
2904             setJSValue(m_out.phi(m_out.int64, simpleResult, unboxedResult));
2905
2906             m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject);
2907             return;
2908         }
2909
2910         case CellUse:
2911         case UntypedUse: {
2912             LValue value;
2913             if (m_node->child1().useKind() == CellUse)
2914                 value = lowCell(m_node->child1());
2915             else
2916                 value = lowJSValue(m_node->child1());
2917
2918             LBasicBlock isCell = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse is cell"));
2919             LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse not string"));
2920             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse continuation"));
2921
2922             LValue isCellPredicate;
2923             if (m_node->child1().useKind() == CellUse)
2924                 isCellPredicate = m_out.booleanTrue;
2925             else
2926                 isCellPredicate = this->isCell(value);
2927             m_out.branch(isCellPredicate, unsure(isCell), unsure(notString));
2928
2929             LBasicBlock lastNext = m_out.appendTo(isCell, notString);
2930             ValueFromBlock simpleResult = m_out.anchor(value);
2931             LValue isStringPredicate;
2932             if (m_node->child1()->prediction() & SpecString) {
2933                 isStringPredicate = m_out.equal(
2934                     m_out.load32(value, m_heaps.JSCell_structureID),
2935                     m_out.constInt32(vm().stringStructure->id()));
2936             } else
2937                 isStringPredicate = m_out.booleanFalse;
2938             m_out.branch(isStringPredicate, unsure(continuation), unsure(notString));
2939
2940             m_out.appendTo(notString, continuation);
2941             LValue operation;
2942             if (m_node->child1().useKind() == CellUse)
2943                 operation = m_out.operation(operationToStringOnCell);
2944             else
2945                 operation = m_out.operation(operationToString);
2946             ValueFromBlock convertedResult = m_out.anchor(vmCall(operation, m_callFrame, value));
2947             m_out.jump(continuation);
2948
2949             m_out.appendTo(continuation, lastNext);
2950             setJSValue(m_out.phi(m_out.int64, simpleResult, convertedResult));
2951             return;
2952         }
2953
2954         default:
2955             LOWERING_FAILED(m_node, "Bad use kind");
2956             return;
2957         }
2958     }
2959
2960     void compileToPrimitive()
2961     {
2962         LValue value = lowJSValue(m_node->child1());
2963
2964         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive cell case"));
2965         LBasicBlock isObjectCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive object case"));
2966         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToPrimitive continuation"));
2967
2968         Vector<ValueFromBlock, 3> results;
2969
2970         results.append(m_out.anchor(value));
2971         m_out.branch(isCell(value), unsure(isCellCase), unsure(continuation));
2972
2973         LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase);
2974         results.append(m_out.anchor(value));
2975         m_out.branch(isObject(value), unsure(isObjectCase), unsure(continuation));
2976
2977         m_out.appendTo(isObjectCase, continuation);
2978         results.append(m_out.anchor(vmCall(
2979             m_out.operation(operationToPrimitive), m_callFrame, value)));
2980         m_out.jump(continuation);
2981
2982         m_out.appendTo(continuation, lastNext);
2983         setJSValue(m_out.phi(m_out.int64, results));
2984     }
2985
2986     void compileMakeRope()
2987     {
2988         LValue kids[3];
2989         unsigned numKids;
2990         kids[0] = lowCell(m_node->child1());
2991         kids[1] = lowCell(m_node->child2());
2992         if (m_node->child3()) {
2993             kids[2] = lowCell(m_node->child3());
2994             numKids = 3;
2995         } else {
2996             kids[2] = 0;
2997             numKids = 2;
2998         }
2999
3000         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MakeRope slow path"));
3001         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MakeRope continuation"));
3002
3003         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3004
3005         MarkedAllocator& allocator =
3006             vm().heap.allocatorForObjectWithImmortalStructureDestructor(sizeof(JSRopeString));
3007
3008         LValue result = allocateCell(
3009             m_out.constIntPtr(&allocator),
3010             vm().stringStructure.get(),
3011             slowPath);
3012
3013         m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSString_value);
3014         for (unsigned i = 0; i < numKids; ++i)
3015             m_out.storePtr(kids[i], result, m_heaps.JSRopeString_fibers[i]);
3016         for (unsigned i = numKids; i < JSRopeString::s_maxInternalRopeLength; ++i)
3017             m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSRopeString_fibers[i]);
3018         LValue flags = m_out.load32(kids[0], m_heaps.JSString_flags);
3019         LValue length = m_out.load32(kids[0], m_heaps.JSString_length);
3020         for (unsigned i = 1; i < numKids; ++i) {
3021             flags = m_out.bitAnd(flags, m_out.load32(kids[i], m_heaps.JSString_flags));
3022             LValue lengthAndOverflow = m_out.addWithOverflow32(
3023                 length, m_out.load32(kids[i], m_heaps.JSString_length));
3024             speculate(Uncountable, noValue(), 0, m_out.extractValue(lengthAndOverflow, 1));
3025             length = m_out.extractValue(lengthAndOverflow, 0);
3026         }
3027         m_out.store32(
3028             m_out.bitAnd(m_out.constInt32(JSString::Is8Bit), flags),
3029             result, m_heaps.JSString_flags);
3030         m_out.store32(length, result, m_heaps.JSString_length);
3031
3032         ValueFromBlock fastResult = m_out.anchor(result);
3033         m_out.jump(continuation);
3034
3035         m_out.appendTo(slowPath, continuation);
3036         ValueFromBlock slowResult;
3037         switch (numKids) {
3038         case 2:
3039             slowResult = m_out.anchor(vmCall(
3040                 m_out.operation(operationMakeRope2), m_callFrame, kids[0], kids[1]));
3041             break;
3042         case 3:
3043             slowResult = m_out.anchor(vmCall(
3044                 m_out.operation(operationMakeRope3), m_callFrame, kids[0], kids[1], kids[2]));
3045             break;
3046         default:
3047             LOWERING_FAILED(m_node, "Bad number of children");
3048             return;
3049             break;
3050         }
3051         m_out.jump(continuation);
3052
3053         m_out.appendTo(continuation, lastNext);
3054         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
3055     }
3056
3057     void compileStringCharAt()
3058     {
3059         LValue base = lowCell(m_node->child1());
3060         LValue index = lowInt32(m_node->child2());
3061         LValue storage = lowStorage(m_node->child3());
3062
3063         LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("GetByVal String fast path"));
3064         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetByVal String slow path"));
3065         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal String continuation"));
3066
3067         m_out.branch(
3068             m_out.aboveOrEqual(
3069                 index, m_out.load32NonNegative(base, m_heaps.JSString_length)),
3070             rarely(slowPath), usually(fastPath));
3071
3072         LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
3073
3074         LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3075
3076         LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 8-bit case"));
3077         LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 16-bit case"));
3078         LBasicBlock bitsContinuation = FTL_NEW_BLOCK(m_out, ("GetByVal String bitness continuation"));
3079         LBasicBlock bigCharacter = FTL_NEW_BLOCK(m_out, ("GetByVal String big character"));
3080
3081         m_out.branch(
3082             m_out.testIsZero32(
3083                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3084                 m_out.constInt32(StringImpl::flagIs8Bit())),
3085             unsure(is16Bit), unsure(is8Bit));
3086
3087         m_out.appendTo(is8Bit, is16Bit);
3088
3089         ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3090             m_out.load8(m_out.baseIndex(
3091                 m_heaps.characters8,
3092                 storage, m_out.zeroExt(index, m_out.intPtr),
3093                 m_state.forNode(m_node->child2()).m_value)),
3094             m_out.int32));
3095         m_out.jump(bitsContinuation);
3096
3097         m_out.appendTo(is16Bit, bigCharacter);
3098
3099         ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3100             m_out.load16(m_out.baseIndex(
3101                 m_heaps.characters16,
3102                 storage, m_out.zeroExt(index, m_out.intPtr),
3103                 m_state.forNode(m_node->child2()).m_value)),
3104             m_out.int32));
3105         m_out.branch(
3106             m_out.aboveOrEqual(char16Bit.value(), m_out.constInt32(0x100)),
3107             rarely(bigCharacter), usually(bitsContinuation));
3108
3109         m_out.appendTo(bigCharacter, bitsContinuation);
3110
3111         Vector<ValueFromBlock, 4> results;
3112         results.append(m_out.anchor(vmCall(
3113             m_out.operation(operationSingleCharacterString),
3114             m_callFrame, char16Bit.value())));
3115         m_out.jump(continuation);
3116
3117         m_out.appendTo(bitsContinuation, slowPath);
3118
3119         LValue character = m_out.phi(m_out.int32, char8Bit, char16Bit);
3120
3121         LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
3122
3123         results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex(
3124             m_heaps.singleCharacterStrings, smallStrings,
3125             m_out.zeroExt(character, m_out.intPtr)))));
3126         m_out.jump(continuation);
3127
3128         m_out.appendTo(slowPath, continuation);
3129
3130         if (m_node->arrayMode().isInBounds()) {
3131             speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue);
3132             results.append(m_out.anchor(m_out.intPtrZero));
3133         } else {
3134             JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3135
3136             if (globalObject->stringPrototypeChainIsSane()) {
3137                 LBasicBlock negativeIndex = FTL_NEW_BLOCK(m_out, ("GetByVal String negative index"));
3138
3139                 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
3140                 m_out.branch(
3141                     m_out.lessThan(index, m_out.int32Zero),
3142                     rarely(negativeIndex), usually(continuation));
3143
3144                 m_out.appendTo(negativeIndex, continuation);
3145             }
3146
3147             results.append(m_out.anchor(vmCall(
3148                 m_out.operation(operationGetByValStringInt), m_callFrame, base, index)));
3149         }
3150
3151         m_out.jump(continuation);
3152
3153         m_out.appendTo(continuation, lastNext);
3154         setJSValue(m_out.phi(m_out.int64, results));
3155     }
3156
3157     void compileStringCharCodeAt()
3158     {
3159         LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 8-bit case"));
3160         LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 16-bit case"));
3161         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt continuation"));
3162
3163         LValue base = lowCell(m_node->child1());
3164         LValue index = lowInt32(m_node->child2());
3165         LValue storage = lowStorage(m_node->child3());
3166
3167         speculate(
3168             Uncountable, noValue(), 0,
3169             m_out.aboveOrEqual(
3170                 index, m_out.load32NonNegative(base, m_heaps.JSString_length)));
3171
3172         LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3173
3174         m_out.branch(
3175             m_out.testIsZero32(
3176                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3177                 m_out.constInt32(StringImpl::flagIs8Bit())),
3178             unsure(is16Bit), unsure(is8Bit));
3179
3180         LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
3181
3182         ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3183             m_out.load8(m_out.baseIndex(
3184                 m_heaps.characters8,
3185                 storage, m_out.zeroExt(index, m_out.intPtr),
3186                 m_state.forNode(m_node->child2()).m_value)),
3187             m_out.int32));
3188         m_out.jump(continuation);
3189
3190         m_out.appendTo(is16Bit, continuation);
3191
3192         ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3193             m_out.load16(m_out.baseIndex(
3194                 m_heaps.characters16,
3195                 storage, m_out.zeroExt(index, m_out.intPtr),
3196                 m_state.forNode(m_node->child2()).m_value)),
3197             m_out.int32));
3198         m_out.jump(continuation);
3199
3200         m_out.appendTo(continuation, lastNext);
3201
3202         setInt32(m_out.phi(m_out.int32, char8Bit, char16Bit));
3203     }
3204
3205     void compileGetByOffset()
3206     {
3207         StorageAccessData& data =
3208             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
3209
3210         setJSValue(loadProperty(
3211             lowStorage(m_node->child1()), data.identifierNumber, data.offset));
3212     }
3213
3214     void compileMultiGetByOffset()
3215     {
3216         LValue base = lowCell(m_node->child1());
3217
3218         MultiGetByOffsetData& data = m_node->multiGetByOffsetData();
3219
3220         Vector<LBasicBlock, 2> blocks(data.variants.size());
3221         for (unsigned i = data.variants.size(); i--;)
3222             blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset case ", i));
3223         LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset fail"));
3224         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset continuation"));
3225
3226         Vector<SwitchCase, 2> cases;
3227         for (unsigned i = data.variants.size(); i--;) {
3228             GetByIdVariant variant = data.variants[i];
3229             for (unsigned j = variant.structureSet().size(); j--;) {
3230                 cases.append(SwitchCase(
3231                     weakStructure(variant.structureSet()[j]), blocks[i], Weight(1)));
3232             }
3233         }
3234         m_out.switchInstruction(
3235             m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3236
3237         LBasicBlock lastNext = m_out.m_nextBlock;
3238
3239         Vector<ValueFromBlock, 2> results;
3240         for (unsigned i = data.variants.size(); i--;) {
3241             m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
3242
3243             GetByIdVariant variant = data.variants[i];
3244             LValue result;
3245             if (variant.specificValue())
3246                 result = m_out.constInt64(JSValue::encode(variant.specificValue()));
3247             else {
3248                 LValue propertyBase;
3249                 if (variant.chain())
3250                     propertyBase = weakPointer(variant.chain()->terminalPrototype());
3251                 else
3252                     propertyBase = base;
3253                 if (!isInlineOffset(variant.offset()))
3254                     propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly);
3255                 result = loadProperty(propertyBase, data.identifierNumber, variant.offset());
3256             }
3257
3258             results.append(m_out.anchor(result));
3259             m_out.jump(continuation);
3260         }
3261
3262         m_out.appendTo(exit, continuation);
3263         terminate(BadCache);
3264         m_out.unreachable();
3265
3266         m_out.appendTo(continuation, lastNext);
3267         setJSValue(m_out.phi(m_out.int64, results));
3268     }
3269
3270     void compilePutByOffset()
3271     {
3272         StorageAccessData& data =
3273             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
3274
3275         storeProperty(
3276             lowJSValue(m_node->child3()),
3277             lowStorage(m_node->child1()), data.identifierNumber, data.offset);
3278     }
3279
3280     void compileMultiPutByOffset()
3281     {
3282         LValue base = lowCell(m_node->child1());
3283         LValue value = lowJSValue(m_node->child2());
3284
3285         MultiPutByOffsetData& data = m_node->multiPutByOffsetData();
3286
3287         Vector<LBasicBlock, 2> blocks(data.variants.size());
3288         for (unsigned i = data.variants.size(); i--;)
3289             blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset case ", i));
3290         LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset fail"));
3291         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset continuation"));
3292
3293         Vector<SwitchCase, 2> cases;
3294         for (unsigned i = data.variants.size(); i--;) {
3295             PutByIdVariant variant = data.variants[i];
3296             cases.append(
3297                 SwitchCase(weakStructure(variant.oldStructure()), blocks[i], Weight(1)));
3298         }
3299         m_out.switchInstruction(
3300             m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3301
3302         LBasicBlock lastNext = m_out.m_nextBlock;
3303
3304         for (unsigned i = data.variants.size(); i--;) {
3305             m_out.appendTo(blocks[i], i  + 1 < data.variants.size() ? blocks[i + 1] : exit);
3306
3307             PutByIdVariant variant = data.variants[i];
3308
3309             LValue storage;
3310             if (variant.kind() == PutByIdVariant::Replace) {
3311                 if (isInlineOffset(variant.offset()))
3312                     storage = base;
3313                 else
3314                     storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
3315             } else {
3316                 m_graph.m_plan.transitions.addLazily(
3317                     codeBlock(), m_node->origin.semantic.codeOriginOwner(),
3318                     variant.oldStructure(), variant.newStructure());
3319
3320                 storage = storageForTransition(
3321                     base, variant.offset(), variant.oldStructure(), variant.newStructure());
3322
3323                 ASSERT(variant.oldStructure()->indexingType() == variant.newStructure()->indexingType());
3324                 ASSERT(variant.oldStructure()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags());
3325                 ASSERT(variant.oldStructure()->typeInfo().type() == variant.newStructure()->typeInfo().type());
3326                 m_out.store32(
3327                     weakStructure(variant.newStructure()), base, m_heaps.JSCell_structureID);
3328             }
3329
3330             storeProperty(value, storage, data.identifierNumber, variant.offset());
3331             m_out.jump(continuation);
3332         }
3333
3334         m_out.appendTo(exit, continuation);
3335         terminate(BadCache);
3336         m_out.unreachable();
3337
3338         m_out.appendTo(continuation, lastNext);
3339     }
3340
3341     void compileGetGlobalVar()
3342     {
3343         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
3344     }
3345
3346     void compilePutGlobalVar()
3347     {
3348         m_out.store64(
3349             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
3350     }
3351
3352     void compileNotifyWrite()
3353     {
3354         VariableWatchpointSet* set = m_node->variableWatchpointSet();
3355
3356         LValue value = lowJSValue(m_node->child1());
3357
3358         LBasicBlock isNotInvalidated = FTL_NEW_BLOCK(m_out, ("NotifyWrite not invalidated case"));
3359         LBasicBlock notifySlow = FTL_NEW_BLOCK(m_out, ("NotifyWrite notify slow case"));
3360         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NotifyWrite continuation"));
3361
3362         LValue state = m_out.load8(m_out.absolute(set->addressOfState()));
3363
3364         m_out.branch(
3365             m_out.equal(state, m_out.constInt8(IsInvalidated)),
3366             usually(continuation), rarely(isNotInvalidated));
3367
3368         LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, notifySlow);
3369
3370         m_out.branch(
3371             m_out.equal(value, m_out.load64(m_out.absolute(set->addressOfInferredValue()))),
3372             unsure(continuation), unsure(notifySlow));
3373
3374         m_out.appendTo(notifySlow, continuation);
3375
3376         vmCall(m_out.operation(operationNotifyWrite), m_callFrame, m_out.constIntPtr(set), value);
3377         m_out.jump(continuation);
3378
3379         m_out.appendTo(continuation, lastNext);
3380     }
3381
3382     void compileGetCallee()
3383     {
3384         setJSValue(m_out.loadPtr(addressFor(JSStack::Callee)));
3385     }
3386
3387     void compileGetScope()
3388     {
3389         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope));
3390     }
3391
3392     void compileGetMyScope()
3393     {
3394         setJSValue(m_out.loadPtr(addressFor(
3395             m_node->origin.semantic.stackOffset() + JSStack::ScopeChain)));
3396     }
3397
3398     void compileSkipScope()
3399     {
3400         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
3401     }
3402
3403     void compileGetClosureRegisters()
3404     {
3405         if (WriteBarrierBase<Unknown>* registers = m_graph.tryGetRegisters(m_node->child1().node())) {
3406             setStorage(m_out.constIntPtr(registers));
3407             return;
3408         }
3409
3410         setStorage(m_out.loadPtr(
3411             lowCell(m_node->child1()), m_heaps.JSVariableObject_registers));
3412     }
3413
3414     void compileGetClosureVar()
3415     {
3416         setJSValue(m_out.load64(
3417             addressFor(lowStorage(m_node->child1()), m_node->varNumber())));
3418     }
3419
3420     void compilePutClosureVar()
3421     {
3422         m_out.store64(
3423             lowJSValue(m_node->child3()),
3424             addressFor(lowStorage(m_node->child2()), m_node->varNumber()));
3425     }
3426
3427     void compileCompareEq()
3428     {
3429         if (m_node->isBinaryUseKind(Int32Use)
3430             || m_node->isBinaryUseKind(Int52RepUse)
3431             || m_node->isBinaryUseKind(DoubleRepUse)
3432             || m_node->isBinaryUseKind(ObjectUse)
3433             || m_node->isBinaryUseKind(BooleanUse)
3434             || m_node->isBinaryUseKind(StringIdentUse)) {
3435             compileCompareStrictEq();
3436             return;
3437         }
3438
3439         if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) {
3440             compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1());
3441             return;
3442         }
3443
3444         if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) {
3445             compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2());
3446             return;
3447         }
3448
3449         if (m_node->isBinaryUseKind(UntypedUse)) {
3450             nonSpeculativeCompare(LLVMIntEQ, operationCompareEq);
3451             return;
3452         }
3453
3454         LOWERING_FAILED(m_node, "Bad use kinds");
3455     }
3456
3457     void compileCompareEqConstant()
3458     {
3459         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
3460         setBoolean(
3461             equalNullOrUndefined(
3462                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
3463     }
3464
3465     void compileCompareStrictEq()
3466     {
3467         if (m_node->isBinaryUseKind(Int32Use)) {
3468             setBoolean(
3469                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3470             return;
3471         }
3472
3473         if (m_node->isBinaryUseKind(Int52RepUse)) {
3474             Int52Kind kind;
3475             LValue left = lowWhicheverInt52(m_node->child1(), kind);
3476             LValue right = lowInt52(m_node->child2(), kind);
3477             setBoolean(m_out.equal(left, right));
3478             return;
3479         }
3480
3481         if (m_node->isBinaryUseKind(DoubleRepUse)) {
3482             setBoolean(
3483                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
3484             return;
3485         }
3486
3487         if (m_node->isBinaryUseKind(StringIdentUse)) {
3488             setBoolean(
3489                 m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2())));
3490             return;
3491         }
3492
3493         if (m_node->isBinaryUseKind(ObjectUse)) {
3494             setBoolean(
3495                 m_out.equal(
3496                     lowNonNullObject(m_node->child1()),
3497                     lowNonNullObject(m_node->child2())));
3498             return;
3499         }
3500
3501         if (m_node->isBinaryUseKind(BooleanUse)) {
3502             setBoolean(
3503                 m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2())));
3504             return;
3505         }
3506
3507         if (m_node->isBinaryUseKind(MiscUse, UntypedUse)
3508             || m_node->isBinaryUseKind(UntypedUse, MiscUse)) {
3509             speculate(m_node->child1());
3510             speculate(m_node->child2());
3511             LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation);
3512             LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation);
3513             setBoolean(m_out.equal(left, right));
3514             return;
3515         }
3516
3517         if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse)
3518             || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) {
3519             Edge leftEdge = m_node->childFor(StringIdentUse);
3520             Edge rightEdge = m_node->childFor(NotStringVarUse);
3521
3522             LValue left = lowStringIdent(leftEdge);
3523             LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation);
3524
3525             LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is cell case"));
3526             LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is string case"));
3527             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar continuation"));
3528
3529             ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
3530             m_out.branch(isCell(rightValue), unsure(isCellCase), unsure(continuation));
3531
3532             LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
3533             ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse);
3534             m_out.branch(isString(rightValue), unsure(isStringCase), unsure(continuation));
3535
3536             m_out.appendTo(isStringCase, continuation);
3537             LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value);
3538             speculateStringIdent(rightEdge, rightValue, right);
3539             ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right));
3540             m_out.jump(continuation);
3541
3542             m_out.appendTo(continuation, lastNext);
3543             setBoolean(m_out.phi(m_out.boolean, notCellResult, notStringResult, isStringResult));
3544             return;
3545         }
3546
3547         LOWERING_FAILED(m_node, "Bad use kinds");
3548     }
3549
3550     void compileCompareStrictEqConstant()
3551     {
3552         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
3553
3554         setBoolean(
3555             m_out.equal(
3556                 lowJSValue(m_node->child1()),
3557                 m_out.constInt64(JSValue::encode(constant))));
3558     }
3559
3560     void compileCompareLess()
3561     {
3562         compare(LLVMIntSLT, LLVMRealOLT, operationCompareLess);
3563     }
3564
3565     void compileCompareLessEq()
3566     {
3567         compare(LLVMIntSLE, LLVMRealOLE, operationCompareLessEq);
3568     }
3569
3570     void compileCompareGreater()
3571     {
3572         compare(LLVMIntSGT, LLVMRealOGT, operationCompareGreater);
3573     }
3574
3575     void compileCompareGreaterEq()
3576     {
3577         compare(LLVMIntSGE, LLVMRealOGE, operationCompareGreaterEq);
3578     }
3579
3580     void compileLogicalNot()
3581     {
3582         setBoolean(m_out.bitNot(boolify(m_node->child1())));
3583     }
3584
3585     void compileCallOrConstruct()
3586     {
3587         int dummyThisArgument = m_node->op() == Call ? 0 : 1;
3588         int numPassedArgs = m_node->numChildren() - 1;
3589         int numArgs = numPassedArgs + dummyThisArgument;
3590
3591         LValue callee = lowJSValue(m_graph.varArgChild(m_node, 0));
3592
3593         unsigned stackmapID = m_stackmapIDs++;
3594
3595         Vector<LValue> arguments;
3596         arguments.append(m_out.constInt64(stackmapID));
3597         arguments.append(m_out.constInt32(sizeOfCall()));
3598         arguments.append(constNull(m_out.ref8));
3599         arguments.append(m_out.constInt32(1 + JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + numArgs));
3600         arguments.append(callee); // callee -> %rax
3601         arguments.append(getUndef(m_out.int64)); // code block
3602         arguments.append(getUndef(m_out.int64)); // scope chain
3603         arguments.append(callee); // callee -> stack
3604         arguments.append(m_out.constInt64(numArgs)); // argument count and zeros for the tag
3605         if (dummyThisArgument)
3606             arguments.append(getUndef(m_out.int64));
3607         for (int i = 0; i < numPassedArgs; ++i)
3608             arguments.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
3609
3610         callPreflight();
3611
3612         LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments);
3613         setInstructionCallingConvention(call, LLVMWebKitJSCallConv);
3614
3615         m_ftlState.jsCalls.append(JSCall(stackmapID, m_node));
3616
3617         setJSValue(call);
3618     }
3619
3620     void compileJump()
3621     {
3622         m_out.jump(lowBlock(m_node->targetBlock()));
3623     }
3624
3625     void compileBranch()
3626     {
3627         m_out.branch(
3628             boolify(m_node->child1()),
3629             WeightedTarget(
3630                 lowBlock(m_node->branchData()->taken.block),
3631                 m_node->branchData()->taken.count),
3632             WeightedTarget(
3633                 lowBlock(m_node->branchData()->notTaken.block),
3634                 m_node->branchData()->notTaken.count));
3635     }
3636
3637     void compileSwitch()
3638     {
3639         SwitchData* data = m_node->switchData();
3640         switch (data->kind) {
3641         case SwitchImm: {
3642             Vector<ValueFromBlock, 2> intValues;
3643             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
3644
3645             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
3646
3647             switch (m_node->child1().useKind()) {
3648             case Int32Use: {
3649                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
3650                 m_out.jump(switchOnInts);
3651                 break;
3652             }
3653
3654             case UntypedUse: {
3655                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
3656                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
3657                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
3658
3659                 LValue boxedValue = lowJSValue(m_node->child1());
3660                 m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt));
3661
3662                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
3663
3664                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
3665                 m_out.jump(switchOnInts);
3666
3667                 m_out.appendTo(isNotInt, isDouble);
3668                 m_out.branch(
3669                     isCellOrMisc(boxedValue),
3670                     usually(lowBlock(data->fallThrough.block)), rarely(isDouble));
3671
3672                 m_out.appendTo(isDouble, innerLastNext);
3673                 LValue doubleValue = unboxDouble(boxedValue);
3674                 LValue intInDouble = m_out.fpToInt32(doubleValue);
3675                 intValues.append(m_out.anchor(intInDouble));
3676                 m_out.branch(
3677                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
3678                     unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block)));
3679                 break;
3680             }
3681
3682             default:
3683                 LOWERING_FAILED(m_node, "Bad use kind");
3684                 return;
3685             }
3686
3687             m_out.appendTo(switchOnInts, lastNext);
3688             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
3689             return;
3690         }
3691
3692         case SwitchChar: {
3693             LValue stringValue;
3694
3695             // FIXME: We should use something other than unsure() for the branch weight
3696             // of the fallThrough block. The main challenge is just that we have multiple
3697             // branches to fallThrough but a single count, so we would need to divvy it up
3698             // among the different lowered branches.
3699             // https://bugs.webkit.org/show_bug.cgi?id=129082
3700
3701             switch (m_node->child1().useKind()) {
3702             case StringUse: {
3703                 stringValue = lowString(m_node->child1());
3704                 break;
3705             }
3706
3707             case UntypedUse: {
3708                 LValue unboxedValue = lowJSValue(m_node->child1());
3709
3710                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
3711                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
3712
3713                 m_out.branch(
3714                     isNotCell(unboxedValue),
3715                     unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase));
3716
3717                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
3718                 LValue cellValue = unboxedValue;
3719                 m_out.branch(
3720                     isNotString(cellValue),
3721                     unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase));
3722
3723                 m_out.appendTo(isStringCase, lastNext);
3724                 stringValue = cellValue;
3725                 break;
3726             }
3727
3728             default:
3729                 LOWERING_FAILED(m_node, "Bad use kind");
3730                 return;
3731             }
3732
3733             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
3734             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
3735             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
3736             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
3737             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
3738             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
3739
3740             m_out.branch(
3741                 m_out.notEqual(
3742                     m_out.load32NonNegative(stringValue, m_heaps.JSString_length),
3743                     m_out.int32One),
3744                 unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1));
3745
3746             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
3747             Vector<ValueFromBlock, 2> values;
3748             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
3749             values.append(m_out.anchor(fastValue));
3750             m_out.branch(m_out.isNull(fastValue), rarely(needResolution), usually(resolved));
3751
3752             m_out.appendTo(needResolution, resolved);
3753             values.append(m_out.anchor(
3754                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
3755             m_out.jump(resolved);
3756
3757             m_out.appendTo(resolved, is8Bit);
3758             LValue value = m_out.phi(m_out.intPtr, values);
3759             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
3760             m_out.branch(
3761                 m_out.testNonZero32(
3762                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
3763                     m_out.constInt32(StringImpl::flagIs8Bit())),
3764                 unsure(is8Bit), unsure(is16Bit));
3765
3766             Vector<ValueFromBlock, 2> characters;
3767             m_out.appendTo(is8Bit, is16Bit);
3768             characters.append(m_out.anchor(
3769                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
3770             m_out.jump(continuation);
3771
3772             m_out.appendTo(is16Bit, continuation);
3773             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
3774             m_out.jump(continuation);
3775
3776             m_out.appendTo(continuation, lastNext);
3777             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
3778             return;
3779         }
3780
3781         case SwitchString:
3782             LOWERING_FAILED(m_node, "Unimplemented");
3783             break;
3784         }
3785
3786         LOWERING_FAILED(m_node, "Bad switch kind");
3787     }
3788
3789     void compileReturn()
3790     {
3791         m_out.ret(lowJSValue(m_node->child1()));
3792     }
3793
3794     void compileForceOSRExit()
3795     {
3796         terminate(InadequateCoverage);
3797     }
3798
3799     void compileThrow()
3800     {
3801         terminate(Uncountable);
3802     }
3803
3804     void compileInvalidationPoint()
3805     {
3806         if (verboseCompilationEnabled())
3807             dataLog("    Invalidation point with availability: ", m_availability, "\n");
3808
3809         m_ftlState.jitCode->osrExit.append(OSRExit(
3810             UncountableInvalidation, InvalidValueFormat, MethodOfGettingAValueProfile(),
3811             m_codeOriginForExitTarget, m_codeOriginForExitProfile,
3812             m_availability.numberOfArguments(), m_availability.numberOfLocals()));
3813         m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
3814
3815         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
3816         OSRExitCompilationInfo& info = m_ftlState.finalizer->osrExit.last();
3817
3818         ExitArgumentList arguments;
3819
3820         buildExitArguments(exit, arguments, FormattedValue(), exit.m_codeOrigin);
3821         callStackmap(exit, arguments);
3822
3823         info.m_isInvalidationPoint = true;
3824     }
3825
3826     void compileCheckArgumentsNotCreated()
3827     {
3828         ASSERT(!isEmptySpeculation(
3829             m_state.variables().operand(
3830                 m_graph.argumentsRegisterFor(m_node->origin.semantic)).m_type));
3831
3832         checkArgumentsNotCreated();
3833     }
3834
3835     void compileIsUndefined()
3836     {
3837         setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
3838     }
3839
3840     void compileIsBoolean()
3841     {
3842         setBoolean(isBoolean(lowJSValue(m_node->child1())));
3843     }
3844
3845     void compileIsNumber()
3846     {
3847         setBoolean(isNumber(lowJSValue(m_node->child1())));
3848     }
3849
3850     void compileIsString()
3851     {
3852         LValue value = lowJSValue(m_node->child1());
3853
3854         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsString cell case"));
3855         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsString continuation"));
3856
3857         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
3858         m_out.branch(isCell(value), unsure(isCellCase), unsure(continuation));
3859
3860         LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
3861         ValueFromBlock cellResult = m_out.anchor(isString(value));
3862         m_out.jump(continuation);
3863
3864         m_out.appendTo(continuation, lastNext);
3865         setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult));
3866     }
3867
3868     void compileIsObject()
3869     {
3870         LValue pointerResult = vmCall(
3871             m_out.operation(operationIsObject), m_callFrame, lowJSValue(m_node->child1()));
3872         setBoolean(m_out.notNull(pointerResult));
3873     }
3874
3875     void compileIsFunction()
3876     {
3877         LValue pointerResult = vmCall(
3878             m_out.operation(operationIsFunction), lowJSValue(m_node->child1()));
3879         setBoolean(m_out.notNull(pointerResult));
3880     }
3881
3882     void compileCheckHasInstance()
3883     {
3884         speculate(
3885             Uncountable, noValue(), 0,
3886             m_out.testIsZero8(
3887                 m_out.load8(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags),
3888                 m_out.constInt8(ImplementsDefaultHasInstance)));
3889     }
3890
3891     void compileInstanceOf()
3892     {
3893         LValue cell;
3894
3895         if (m_node->child1().useKind() == UntypedUse)
3896             cell = lowJSValue(m_node->child1());
3897         else
3898             cell = lowCell(m_node->child1());
3899
3900         LValue prototype = lowCell(m_node->child2());
3901
3902         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("InstanceOf cell case"));
3903         LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("InstanceOf loop"));
3904         LBasicBlock notYetInstance = FTL_NEW_BLOCK(m_out, ("InstanceOf not yet instance"));
3905         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("InstanceOf continuation"));
3906
3907         LValue condition;
3908         if (m_node->child1().useKind() == UntypedUse)
3909             condition = isCell(cell);
3910         else
3911             condition = m_out.booleanTrue;
3912
3913         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
3914         m_out.branch(condition, unsure(isCellCase), unsure(continuation));
3915
3916         LBasicBlock lastNext = m_out.appendTo(isCellCase, loop);
3917
3918         speculate(BadType, noValue(), 0, isNotObject(prototype));
3919
3920         ValueFromBlock originalValue = m_out.anchor(cell);
3921         m_out.jump(loop);
3922
3923         m_out.appendTo(loop, notYetInstance);
3924         LValue value = m_out.phi(m_out.int64, originalValue);
3925         LValue structure = loadStructure(value);
3926         LValue currentPrototype = m_out.load64(structure, m_heaps.Structure_prototype);
3927         ValueFromBlock isInstanceResult = m_out.anchor(m_out.booleanTrue);
3928         m_out.branch(
3929             m_out.equal(currentPrototype, prototype),
3930             unsure(continuation), unsure(notYetInstance));
3931
3932         m_out.appendTo(notYetInstance, continuation);
3933         ValueFromBlock notInstanceResult = m_out.anchor(m_out.booleanFalse);
3934         addIncoming(value, m_out.anchor(currentPrototype));
3935         m_out.branch(isCell(currentPrototype), unsure(loop), unsure(continuation));
3936
3937         m_out.appendTo(continuation, lastNext);
3938         setBoolean(
3939             m_out.phi(m_out.boolean, notCellResult, isInstanceResult, notInstanceResult));
3940     }
3941
3942     void compileCountExecution()
3943     {
3944         TypedPointer counter = m_out.absolute(m_node->executionCounter()->address());
3945         m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter);
3946     }
3947
3948     void compileStoreBarrier()
3949     {
3950         emitStoreBarrier(lowCell(m_node->child1()));
3951     }
3952
3953     void compileStoreBarrierWithNullCheck()
3954     {
3955 #if ENABLE(GGC)
3956         LBasicBlock isNotNull = FTL_NEW_BLOCK(m_out, ("Store barrier with null check value not null"));
3957         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation"));
3958
3959         LValue base = lowJSValue(m_node->child1());
3960         m_out.branch(m_out.isZero64(base), unsure(continuation), unsure(isNotNull));
3961         LBasicBlock lastNext = m_out.appendTo(isNotNull, continuation);
3962         emitStoreBarrier(base);
3963         m_out.appendTo(continuation, lastNext);
3964 #else
3965         speculate(m_node->child1());
3966 #endif
3967     }
3968
3969     LValue didOverflowStack()
3970     {
3971         // This does a very simple leaf function analysis. The invariant of FTL call
3972         // frames is that the caller had already done enough of a stack check to
3973         // prove that this call frame has enough stack to run, and also enough stack
3974         // to make runtime calls. So, we only need to stack check when making calls
3975         // to other JS functions. If we don't find such calls then we don't need to
3976         // do any stack checks.
3977
3978         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
3979             BasicBlock* block = m_graph.block(blockIndex);
3980             if (!block)
3981                 continue;
3982
3983             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
3984                 Node* node = block->at(nodeIndex);
3985
3986                 switch (node->op()) {
3987                 case GetById:
3988                 case PutById:
3989                 case Call:
3990                 case Construct:
3991                     return m_out.below(
3992                         m_callFrame,
3993                         m_out.loadPtr(
3994                             m_out.absolute(vm().addressOfFTLStackLimit())));
3995
3996                 default:
3997                     break;
3998                 }
3999             }
4000         }
4001
4002         return m_out.booleanFalse;
4003     }
4004
4005     LValue numberOrNotCellToInt32(Edge edge, LValue value)
4006     {
4007         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 int case"));
4008         LBasicBlock notIntCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not int case"));
4009         LBasicBlock doubleCase = 0;
4010         LBasicBlock notNumberCase = 0;
4011         if (edge.useKind() == NotCellUse) {
4012             doubleCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 double case"));
4013             notNumberCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not number case"));
4014         }
4015         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ValueToInt32 continuation"));
4016
4017         Vector<ValueFromBlock> results;
4018
4019         m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase));
4020
4021         LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase);
4022         results.append(m_out.anchor(unboxInt32(value)));
4023         m_out.jump(continuation);
4024
4025         if (edge.useKind() == NumberUse) {
4026             m_out.appendTo(notIntCase, continuation);
4027             FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value));
4028             results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
4029             m_out.jump(continuation);
4030         } else {
4031             m_out.appendTo(notIntCase, doubleCase);
4032             m_out.branch(isCellOrMisc(value), unsure(notNumberCase), unsure(doubleCase));
4033
4034             m_out.appendTo(doubleCase, notNumberCase);
4035             results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
4036             m_out.jump(continuation);
4037
4038             m_out.appendTo(notNumberCase, continuation);
4039
4040             FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCell, isCell(value));
4041
4042             LValue specialResult = m_out.select(
4043                 m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))),
4044                 m_out.int32One, m_out.int32Zero);
4045             results.append(m_out.anchor(specialResult));
4046             m_out.jump(continuation);
4047         }
4048
4049         m_out.appendTo(continuation, lastNext);
4050         return m_out.phi(m_out.int32, results);
4051     }
4052
4053     LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset)
4054     {
4055         return m_out.load64(addressOfProperty(storage, identifierNumber, offset));
4056     }
4057
4058     void storeProperty(
4059         LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset)
4060     {
4061         m_out.store64(value, addressOfProperty(storage, identifierNumber, offset));
4062     }
4063
4064     TypedPointer addressOfProperty(
4065         LValue storage, unsigned identifierNumber, PropertyOffset offset)
4066     {
4067         return m_out.address(
4068             m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset));
4069     }
4070
4071     LValue storageForTransition(
4072         LValue object, PropertyOffset offset,
4073         Structure* previousStructure, Structure* nextStructure)
4074     {
4075         if (isInlineOffset(offset))
4076             return object;
4077
4078         if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity())
4079             return m_out.loadPtr(object, m_heaps.JSObject_butterfly);
4080
4081         LValue result;
4082         if (!previousStructure->outOfLineCapacity())
4083             result = allocatePropertyStorage(object, previousStructure);
4084         else {
4085             result = reallocatePropertyStorage(
4086                 object, m_out.loadPtr(object, m_heaps.JSObject_butterfly),
4087                 previousStructure, nextStructure);
4088         }
4089
4090         emitStoreBarrier(object);
4091
4092         return result;
4093     }
4094
4095     LValue allocatePropertyStorage(LValue object, Structure* previousStructure)
4096     {
4097         if (previousStructure->couldHaveIndexingHeader()) {
4098             return vmCall(
4099                 m_out.operation(
4100                     operationReallocateButterflyToHavePropertyStorageWithInitialCapacity),
4101                 m_callFrame, object);
4102         }
4103
4104         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorage slow path"));
4105         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorage continuation"));
4106
4107         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
4108
4109         LValue endOfStorage = allocateBasicStorageAndGetEnd(
4110             m_out.constIntPtr(initialOutOfLineCapacity * sizeof(JSValue)), slowPath);
4111
4112         ValueFromBlock fastButterfly = m_out.anchor(
4113             m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage));
4114
4115         m_out.jump(continuation);
4116
4117         m_out.appendTo(slowPath, continuation);
4118
4119         ValueFromBlock slowButterfly = m_out.anchor(vmCall(
4120             m_out.operation(operationAllocatePropertyStorageWithInitialCapacity), m_callFrame));
4121
4122         m_out.jump(continuation);
4123
4124         m_out.appendTo(continuation, lastNext);
4125
4126         LValue result = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
4127         m_out.storePtr(result, object, m_heaps.JSObject_butterfly);
4128
4129         return result;
4130     }
4131
4132     LValue reallocatePropertyStorage(
4133         LValue object, LValue oldStorage, Structure* previous, Structure* next)
4134     {
4135         size_t oldSize = previous->outOfLineCapacity() * sizeof(JSValue);
4136         size_t newSize = oldSize * outOfLineGrowthFactor;
4137
4138         ASSERT_UNUSED(next, newSize == next->outOfLineCapacity() * sizeof(JSValue));
4139
4140         if (previous->couldHaveIndexingHeader()) {
4141             LValue newAllocSize = m_out.constInt64(newSize / sizeof(JSValue));
4142             return vmCall(m_out.operation(operationReallocateButterflyToGrowPropertyStorage), m_callFrame, object, newAllocSize);
4143         }
4144
4145         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("reallocatePropertyStorage slow path"));
4146         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("reallocatePropertyStorage continuation"));
4147         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
4148
4149         LValue endOfStorage =
4150             allocateBasicStorageAndGetEnd(m_out.constIntPtr(newSize), slowPath);
4151
4152         ValueFromBlock fastButterfly = m_out.anchor(m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage));
4153
4154         m_out.jump(continuation);
4155
4156         m_out.appendTo(slowPath, continuation);
4157
4158         LValue newAllocSize = m_out.constInt64(newSize / sizeof(JSValue));
4159
4160         LValue storageLocation = vmCall(m_out.operation(operationAllocatePropertyStorage), m_callFrame, newAllocSize);
4161
4162         ValueFromBlock slowButterfly = m_out.anchor(storageLocation);
4163
4164         m_out.jump(continuation);
4165
4166         m_out.appendTo(continuation, lastNext);
4167
4168         LValue result = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
4169
4170         ptrdiff_t headerSize = -sizeof(JSValue) - sizeof(void *);
4171         ptrdiff_t endStorage = headerSize - static_cast<ptrdiff_t>(oldSize);
4172
4173         for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) {
4174             LValue loaded =
4175                 m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset));
4176             m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset));
4177         }
4178
4179         m_out.storePtr(result, m_out.address(object, m_heaps.JSObject_butterfly));
4180
4181         return result;
4182     }
4183
4184     LValue getById(LValue base)
4185     {
4186         StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
4187
4188         // Arguments: id, bytes, target, numArgs, args...
4189         unsigned stackmapID = m_stackmapIDs++;
4190
4191         if (Options::verboseCompilation())
4192             dataLog("    Emitting GetById patchpoint with stackmap #", stackmapID, "\n");
4193
4194         LValue call = m_out.call(
4195             m_out.patchpointInt64Intrinsic(),
4196             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfGetById()),
4197             constNull(m_out.ref8), m_out.constInt32(1), base);
4198         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
4199
4200         m_ftlState.getByIds.append(GetByIdDescriptor(stackmapID, m_node->origin.semantic, uid));
4201
4202         return call;
4203     }
4204
4205     TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge)
4206     {
4207         return m_out.baseIndex(
4208             heap, storage, m_out.zeroExt(index, m_out.intPtr),
4209             m_state.forNode(edge).m_value);
4210     }
4211
4212     void compare(
4213         LIntPredicate intCondition, LRealPredicate realCondition,
4214         S_JITOperation_EJJ helperFunction)
4215     {
4216         if (m_node->isBinaryUseKind(Int32Use)) {
4217             LValue left = lowInt32(m_node->child1());
4218             LValue right = lowInt32(m_node->child2());
4219             setBoolean(m_out.icmp(intCondition, left, right));
4220             return;
4221         }
4222
4223         if (m_node->isBinaryUseKind(Int52RepUse)) {
4224             Int52Kind kind;
4225             LValue left = lowWhicheverInt52(m_node->child1(), kind);
4226             LValue right = lowInt52(m_node->child2(), kind);
4227             setBoolean(m_out.icmp(intCondition, left, right));
4228             return;
4229         }
4230
4231         if (m_node->isBinaryUseKind(DoubleRepUse)) {
4232             LValue left = lowDouble(m_node->child1());
4233             LValue right = lowDouble(m_node->child2());
4234             setBoolean(m_out.fcmp(realCondition, left, right));
4235             return;
4236         }
4237
4238         if (m_node->isBinaryUseKind(UntypedUse)) {
4239             nonSpeculativeCompare(intCondition, helperFunction);
4240             return;
4241         }
4242
4243         LOWERING_FAILED(m_node, "Bad use kinds");
4244     }
4245
4246     void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild)
4247     {
4248         LValue rightCell = lowCell(rightChild);
4249         LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation);
4250
4251         speculateTruthyObject(rightChild, rightCell, SpecObject);
4252
4253         LBasicBlock leftCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left cell case"));
4254         LBasicBlock leftNotCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left not cell case"));
4255         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject continuation"));
4256
4257         m_out.branch(isCell(leftValue), unsure(leftCellCase), unsure(leftNotCellCase));
4258
4259         LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase);
4260         speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCell));
4261         ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue));
4262         m_out.jump(continuation);
4263
4264         m_out.appendTo(leftNotCellCase, continuation);
4265         FTL_TYPE_CHECK(
4266             jsValueValue(leftValue), leftChild, SpecOther | SpecCell, isNotOther(leftValue));
4267         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4268         m_out.jump(continuation);
4269
4270         m_out.appendTo(continuation, lastNext);
4271         setBoolean(m_out.phi(m_out.boolean, cellResult, notCellResult));
4272     }
4273
4274     void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter)
4275     {
4276         if (masqueradesAsUndefinedWatchpointIsStillValid()) {
4277             FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
4278             return;
4279         }
4280
4281         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
4282         FTL_TYPE_CHECK(
4283             jsValueValue(cell), edge, filter,
4284             m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())));
4285         speculate(
4286             BadType, jsValueValue(cell), edge.node(),
4287             m_out.testNonZero8(
4288                 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
4289                 m_out.constInt8(MasqueradesAsUndefined)));
4290     }
4291
4292     void nonSpeculativeCompare(LIntPredicate intCondition, S_JITOperation_EJJ helperFunction)
4293     {
4294         LValue left = lowJSValue(m_node->child1());
4295         LValue right = lowJSValue(m_node->child2());
4296
4297         LBasicBlock leftIsInt = FTL_NEW_BLOCK(m_out, ("CompareEq untyped left is int"));
4298         LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped fast path"));
4299         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped slow path"));
4300         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEq untyped continuation"));
4301
4302         m_out.branch(isNotInt32(left), rarely(slowPath), usually(leftIsInt));
4303
4304         LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath);
4305         m_out.branch(isNotInt32(right), rarely(slowPath), usually(fastPath));
4306
4307         m_out.appendTo(fastPath, slowPath);
4308         ValueFromBlock fastResult = m_out.anchor(
4309             m_out.icmp(intCondition, unboxInt32(left), unboxInt32(right)));
4310         m_out.jump(continuation);
4311
4312         m_out.appendTo(slowPath, continuation);
4313         ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
4314             m_out.operation(helperFunction), m_callFrame, left, right)));
4315         m_out.jump(continuation);
4316
4317         m_out.appendTo(continuation, lastNext);
4318         setBoolean(m_out.phi(m_out.boolean, fastResult, slowResult));
4319     }
4320
4321     LValue allocateCell(LValue allocator, Structure* structure, LBasicBlock slowPath)
4322     {
4323         LBasicBlock success = FTL_NEW_BLOCK(m_out, ("object allocation success"));
4324
4325         LValue result = m_out.loadPtr(
4326             allocator, m_heaps.MarkedAllocator_freeListHead);
4327
4328         m_out.branch(m_out.notNull(result), usually(success), rarely(slowPath));
4329
4330         m_out.appendTo(success);
4331
4332         m_out.storePtr(
4333             m_out.loadPtr(result, m_heaps.JSCell_freeListNext),
4334             allocator, m_heaps.MarkedAllocator_freeListHead);
4335
4336         m_out.store32(m_out.constInt32(structure->id()), result, m_heaps.JSCell_structureID);
4337         m_out.store8(m_out.constInt8(structure->indexingType()), result, m_heaps.JSCell_indexingType);
4338         m_out.store8(m_out.constInt8(structure->typeInfo().type()), result, m_heaps.JSCell_typeInfoType);
4339         m_out.store8(m_out.constInt8(structure->typeInfo().inlineTypeFlags()), result, m_heaps.JSCell_typeInfoFlags);
4340         m_out.store8(m_out.constInt8(JSCell::NotMarked), result, m_heaps.JSCell_gcData);
4341
4342         return result;
4343     }
4344
4345     LValue allocateObject(
4346         LValue allocator, Structure* structure, LValue butterfly, LBasicBlock slowPath)
4347     {
4348         LValue result = allocateCell(allocator, structure, slowPath);
4349         m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly);
4350         return result;
4351     }
4352
4353     template<typename ClassType>
4354     LValue allocateObject(Structure* structure, LValue butterfly, LBasicBlock slowPath)
4355     {
4356         MarkedAllocator* allocator;
4357         size_t size = ClassType::allocationSize(0);
4358         if (ClassType::needsDestruction && ClassType::hasImmortalStructure)
4359             allocator = &vm().heap.allocatorForObjectWithImmortalStructureDestructor(size);
4360         else if (ClassType::needsDestruction)
4361             allocator = &vm().heap.allocatorForObjectWithNormalDestructor(size);
4362         else
4363             allocator = &vm().heap.allocatorForObjectWithoutDestructor(size);
4364         return allocateObject(m_out.constIntPtr(allocator), structure, butterfly, slowPath);
4365     }
4366
4367     // Returns a pointer to the end of the allocation.
4368     LValue allocateBasicStorageAndGetEnd(LValue size, LBasicBlock slowPath)
4369     {
4370         CopiedAllocator& allocator = vm().heap.storageAllocator();
4371
4372         LBasicBlock success = FTL_NEW_BLOCK(m_out, ("storage allocation success"));
4373
4374         LValue remaining = m_out.loadPtr(m_out.absolute(&allocator.m_currentRemaining));
4375         LValue newRemaining = m_out.sub(remaining, size);
4376
4377         m_out.branch(
4378             m_out.lessThan(newRemaining, m_out.intPtrZero),
4379             rarely(slowPath), usually(success));
4380
4381         m_out.appendTo(success);
4382
4383         m_out.storePtr(newRemaining, m_out.absolute(&allocator.m_currentRemaining));
4384         return m_out.sub(
4385             m_out.loadPtr(m_out.absolute(&allocator.m_currentPayloadEnd)), newRemaining);
4386     }
4387
4388     struct ArrayValues {
4389         ArrayValues()
4390             : array(0)
4391             , butterfly(0)
4392         {
4393         }
4394
4395         ArrayValues(LValue array, LValue butterfly)
4396             : array(array)
4397             , butterfly(butterfly)
4398         {
4399         }
4400
4401         LValue array;
4402         LValue butterfly;
4403     };
4404     ArrayValues allocateJSArray(
4405         Structure* structure, unsigned numElements, LBasicBlock slowPath)
4406     {
4407         ASSERT(
4408             hasUndecided(structure->indexingType())
4409             || hasInt32(structure->indexingType())
4410             || hasDouble(structure->indexingType())
4411             || hasContiguous(structure->indexingType()));
4412
4413         unsigned vectorLength = std::max(BASE_VECTOR_LEN, numElements);
4414
4415         LValue endOfStorage = allocateBasicStorageAndGetEnd(
4416             m_out.constIntPtr(sizeof(JSValue) * vectorLength + sizeof(IndexingHeader)),
4417             slowPath);
4418
4419         LValue butterfly = m_out.sub(
4420             endOfStorage, m_out.constIntPtr(sizeof(JSValue) * vectorLength));
4421
4422         LValue object = allocateObject<JSArray>(
4423             structure, butterfly, slowPath);
4424
4425         m_out.store32(m_out.constInt32(numElements), butterfly, m_heaps.Butterfly_publicLength);
4426         m_out.store32(m_out.constInt32(vectorLength), butterfly, m_heaps.Butterfly_vectorLength);
4427
4428         if (hasDouble(structure->indexingType())) {
4429             for (unsigned i = numElements; i < vectorLength; ++i) {
4430                 m_out.store64(
4431                     m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
4432                     butterfly, m_heaps.indexedDoubleProperties[i]);
4433             }
4434         }
4435
4436         return ArrayValues(object, butterfly);
4437     }
4438
4439     ArrayValues allocateJSArray(Structure* structure, unsigned numElements)
4440     {
4441         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("JSArray allocation slow path"));
4442         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("JSArray allocation continuation"));
4443
4444         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
4445
4446         ArrayValues fastValues = allocateJSArray(structure, numElements, slowPath);
4447         ValueFromBlock fastArray = m_out.anchor(fastValues.array);
4448         ValueFromBlock fastButterfly = m_out.anchor(fastValues.butterfly);
4449
4450         m_out.jump(continuation);
4451
4452         m_out.appendTo(slowPath, continuation);
4453
4454         ValueFromBlock slowArray = m_out.anchor(vmCall(
4455             m_out.operation(operationNewArrayWithSize), m_callFrame,
4456             m_out.constIntPtr(structure), m_out.constInt32(numElements)));
4457         ValueFromBlock slowButterfly = m_out.anchor(
4458             m_out.loadPtr(slowArray.value(), m_heaps.JSObject_butterfly));
4459
4460         m_out.jump(continuation);
4461
4462         m_out.appendTo(continuation, lastNext);
4463
4464         return ArrayValues(
4465             m_out.phi(m_out.intPtr, fastArray, slowArray),
4466             m_out.phi(m_out.intPtr, fastButterfly, slowButterfly));
4467     }
4468
4469     LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode, LValue base)
4470     {
4471         if (JSArrayBufferView* view = m_graph.tryGetFoldableView(baseEdge.node(), arrayMode))
4472             return m_out.constInt32(view->length());
4473         return m_out.load32NonNegative(base, m_heaps.JSArrayBufferView_length);
4474     }
4475
4476     LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode)
4477     {
4478         return typedArrayLength(baseEdge, arrayMode, lowCell(baseEdge));
4479     }
4480
4481     LValue boolify(Edge edge)
4482     {
4483         switch (edge.useKind()) {
4484         case BooleanUse:
4485             return lowBoolean(m_node->child1());
4486         case Int32Use:
4487             return m_out.notZero32(lowInt32(m_node->child1()));
4488         case DoubleRepUse:
4489             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
4490         case ObjectOrOtherUse:
4491             return m_out.bitNot(
4492                 equalNullOrUndefined(
4493                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
4494                     ManualOperandSpeculation));
4495         case StringUse: {
4496             LValue stringValue = lowString(m_node->child1());
4497             LValue length = m_out.load32NonNegative(stringValue, m_heaps.JSString_length);
4498             return m_out.notEqual(length, m_out.int32Zero);
4499         }
4500         case UntypedUse: {
4501             LValue value = lowJSValue(m_node->child1());
4502
4503             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped slow case"));
4504             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped fast case"));
4505             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Boolify untyped continuation"));
4506
4507             m_out.branch(isNotBoolean(value), rarely(slowCase), usually(fastCase));
4508
4509             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
4510             ValueFromBlock fastResult = m_out.anchor(unboxBoolean(value));
4511             m_out.jump(continuation);
4512
4513             m_out.appendTo(slowCase, continuation);
4514             ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
4515                 m_out.operation(operationConvertJSValueToBoolean), m_callFrame, value)));
4516             m_out.jump(continuation);
4517
4518             m_out.appendTo(continuation, lastNext);
4519             return m_out.phi(m_out.boolean, fastResult, slowResult);
4520         }
4521         default:
4522             LOWERING_FAILED(m_node, "Bad use kind");
4523             return 0;
4524         }
4525     }
4526
4527     enum StringOrObjectMode {
4528         AllCellsAreFalse,
4529         CellCaseSpeculatesObject
4530     };
4531     enum EqualNullOrUndefinedMode {
4532         EqualNull,
4533         EqualUndefined,
4534         EqualNullOrUndefined,
4535         SpeculateNullOrUndefined
4536     };
4537     LValue equalNullOrUndefined(
4538         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
4539         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
4540     {
4541         bool validWatchpoint = masqueradesAsUndefinedWatchpointIsStillValid();
4542
4543         LValue value = lowJSValue(edge, operandMode);
4544
4545         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
4546         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
4547         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
4548
4549         m_out.branch(isNotCell(value), unsure(primitiveCase), unsure(cellCase));
4550
4551         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
4552
4553         Vector<ValueFromBlock, 3> results;
4554
4555         switch (cellMode) {
4556         case AllCellsAreFalse:
4557             break;
4558         case CellCaseSpeculatesObject:
4559             FTL_TYPE_CHECK(
4560                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
4561                 m_out.equal(
4562                     m_out.load32(value, m_heaps.JSCell_structureID),
4563                     m_out.constInt32(vm().stringStructure->id())));
4564             break;
4565         }
4566
4567         if (validWatchpoint) {
4568             results.append(m_out.anchor(m_out.booleanFalse));
4569             m_out.jump(continuation);
4570         } else {
4571             LBasicBlock masqueradesCase =
4572                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
4573
4574             results.append(m_out.anchor(m_out.booleanFalse));
4575
4576             m_out.branch(
4577                 m_out.testNonZero8(
4578                     m_out.load8(value, m_heaps.JSCell_typeInfoFlags),
4579                     m_out.constInt8(MasqueradesAsUndefined)),
4580                 rarely(masqueradesCase), usually(continuation));
4581
4582             m_out.appendTo(masqueradesCase, primitiveCase);
4583
4584             LValue structure = loadStructure(value);
4585
4586             results.append(m_out.anchor(
4587                 m_out.equal(
4588                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)),
4589                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
4590             m_out.jump(continuation);
4591         }
4592
4593         m_out.appendTo(primitiveCase, continuation);
4594
4595         LValue primitiveResult;
4596         switch (primitiveMode) {
4597         case EqualNull:
4598             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
4599             break;
4600         case EqualUndefined:
4601             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
4602             break;
4603         case EqualNullOrUndefined:
4604             primitiveResult = isOther(value);
4605             break;
4606         case SpeculateNullOrUndefined:
4607             FTL_TYPE_CHECK(
4608                 jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
4609             primitiveResult = m_out.booleanTrue;
4610             break;
4611         }
4612         results.append(m_out.anchor(primitiveResult));
4613         m_out.jump(continuation);
4614
4615         m_out.appendTo(continuation, lastNext);
4616
4617         return m_out.phi(m_out.boolean, results);
4618     }
4619
4620     template<typename FunctionType>
4621     void contiguousPutByValOutOfBounds(
4622         FunctionType slowPathFunction, LValue base, LValue storage, LValue index, LValue value,
4623         LBasicBlock continuation)
4624     {
4625         LValue isNotInBounds = m_out.aboveOrEqual(
4626             index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength));
4627         if (!m_node->arrayMode().isInBounds()) {
4628             LBasicBlock notInBoundsCase =
4629                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
4630             LBasicBlock performStore =
4631                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
4632
4633             m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore));
4634
4635             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
4636
4637             LValue isOutOfBounds = m_out.aboveOrEqual(
4638                 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength));
4639
4640             if (!m_node->arrayMode().isOutOfBounds())
4641                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
4642             else {
4643                 LBasicBlock outOfBoundsCase =
4644                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
4645                 LBasicBlock holeCase =
4646                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
4647
4648                 m_out.branch(isOutOfBounds, unsure(outOfBoundsCase), unsure(holeCase));
4649
4650                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
4651
4652                 vmCall(
4653                     m_out.operation(slowPathFunction),
4654                     m_callFrame, base, index, value);
4655
4656                 m_out.jump(continuation);
4657
4658                 m_out.appendTo(holeCase, innerLastNext);
4659             }
4660
4661             m_out.store32(
4662                 m_out.add(index, m_out.int32One),
4663                 storage, m_heaps.Butterfly_publicLength);
4664
4665             m_out.jump(performStore);
4666             m_out.appendTo(performStore, lastNext);
4667         }
4668     }
4669
4670     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
4671     {
4672         Vector<SwitchCase> cases;
4673         for (unsigned i = 0; i < data->cases.size(); ++i) {
4674             cases.append(SwitchCase(
4675                 constInt(type, data->cases[i].value.switchLookupValue()),
4676                 lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count)));
4677         }
4678
4679         m_out.switchInstruction(
4680             switchValue, cases,
4681             lowBlock(data->fallThrough.block), Weight(data->fallThrough.count));
4682     }
4683
4684     LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
4685     {
4686         LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough"));
4687         LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange"));
4688         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath"));
4689         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("doubleToInt32 continuation"));
4690
4691         Vector<ValueFromBlock, 2> results;
4692
4693         m_out.branch(
4694             m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)),
4695             unsure(greatEnough), unsure(slowPath));
4696
4697         LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
4698         m_out.branch(
4699             m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)),
4700             unsure(withinRange), unsure(slowPath));
4701
4702         m_out.appendTo(withinRange, slowPath);
4703         LValue fastResult;
4704         if (isSigned)
4705             fastResult = m_out.fpToInt32(doubleValue);
4706         else
4707             fastResult = m_out.fpToUInt32(doubleValue);
4708         results.append(m_out.anchor(fastResult));
4709         m_out.jump(continuation);
4710
4711         m_out.appendTo(slowPath, continuation);
4712         results.append(m_out.anchor(m_out.call(m_out.operation(toInt32), doubleValue)));
4713         m_out.jump(continuation);
4714
4715         m_out.appendTo(continuation, lastNext);
4716         return m_out.phi(m_out.int32, results);
4717     }
4718
4719     LValue doubleToInt32(LValue doubleValue)
4720     {
4721         if (Output::hasSensibleDoubleToInt())
4722             return sensibleDoubleToInt32(doubleValue);
4723
4724         double limit = pow(2, 31) - 1;
4725         return doubleToInt32(doubleValue, -limit, limit);
4726     }
4727
4728     LValue sensibleDoubleToInt32(LValue doubleValue)
4729     {
4730         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 slow path"));
4731         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 continuation"));
4732
4733         ValueFromBlock fastResult = m_out.anchor(
4734             m_out.sensibleDoubleToInt(doubleValue));
4735         m_out.branch(
4736             m_out.equal(fastResult.value(), m_out.constInt32(0x80000000)),
4737             rarely(slowPath), usually(continuation));
4738
4739         LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
4740         ValueFromBlock slowResult = m_out.anchor(
4741             m_out.call(m_out.operation(toInt32), doubleValue));
4742         m_out.jump(continuation);
4743
4744         m_out.appendTo(continuation, lastNext);
4745         return m_out.phi(m_out.int32, fastResult, slowResult);
4746     }
4747
4748     void checkArgumentsNotCreated()
4749     {
4750         CodeOrigin codeOrigin = m_node->origin.semantic;
4751         VirtualRegister argumentsRegister = m_graph.argumentsRegisterFor(codeOrigin);
4752         if (isEmptySpeculation(m_state.variables().operand(argumentsRegister).m_type))
4753             return;
4754
4755         VirtualRegister argsReg = m_graph.machineArgumentsRegisterFor(codeOrigin);
4756         speculate(
4757             ArgumentsEscaped, noValue(), 0,
4758             m_out.notZero64(m_out.load64(addressFor(argsReg))));
4759     }
4760
4761     void speculate(
4762         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
4763     {
4764         appendOSRExit(kind, lowValue, highValue, failCondition);
4765     }
4766
4767     void terminate(ExitKind kind)
4768     {
4769         speculate(kind, noValue(), 0, m_out.booleanTrue);
4770     }
4771
4772     void typeCheck(
4773         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
4774         LValue failCondition)
4775     {
4776         appendTypeCheck(lowValue, highValue, typesPassedThrough, failCondition);
4777     }
4778
4779     void appendTypeCheck(
4780         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
4781         LValue failCondition)
4782     {
4783         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
4784             return;
4785         ASSERT(mayHaveTypeCheck(highValue.useKind()));
4786         appendOSRExit(BadType, lowValue, highValue.node(), failCondition);
4787         m_interpreter.filter(highValue, typesPassedThrough);
4788     }
4789
4790     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4791     {
4792         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
4793
4794         if (edge->hasConstant()) {
4795             JSValue value = m_graph.valueOfJSConstant(edge.node());
4796             if (!value.isInt32()) {
4797                 terminate(Uncountable);
4798                 return m_out.int32Zero;
4799             }
4800             return m_out.constInt32(value.asInt32());
4801         }
4802
4803         LoweredNodeValue value = m_int32Values.get(edge.node());
4804         if (isValid(value))
4805             return value.value();
4806
4807         value = m_strictInt52Values.get(edge.node());
4808         if (isValid(value))
4809             return strictInt52ToInt32(edge, value.value());
4810
4811         value = m_int52Values.get(edge.node());
4812         if (isValid(value))
4813             return strictInt52ToInt32(edge, int52ToStrictInt52(value.value()));
4814
4815         value = m_jsValueValues.get(edge.node());
4816         if (isValid(value)) {
4817             LValue boxedResult = value.value();
4818             FTL_TYPE_CHECK(
4819                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
4820             LValue result = unboxInt32(boxedResult);
4821             setInt32(edge.node(), result);
4822             return result;
4823         }
4824
4825         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
4826         terminate(Uncountable);
4827         return m_out.int32Zero;
4828     }
4829
4830     enum Int52Kind { StrictInt52, Int52 };
4831     LValue lowInt52(Edge edge, Int52Kind kind)
4832     {
4833         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
4834
4835         LoweredNodeValue value;
4836
4837         switch (kind) {
4838         case Int52:
4839             value = m_int52Values.get(edge.node());
4840             if (isValid(value))
4841                 return value.value();
4842
4843             value = m_strictInt52Values.get(edge.node());
4844             if (isValid(value))
4845                 return strictInt52ToInt52(value.value());
4846             break;
4847
4848         case StrictInt52:
4849             value = m_strictInt52Values.get(edge.node());
4850             if (isValid(value))
4851                 return value.value();
4852
4853             value = m_int52Values.get(edge.node());
4854             if (isValid(value))
4855                 return int52ToStrictInt52(value.value());
4856             break;
4857         }
4858
4859         RELEASE_ASSERT(!m_state.forNode(edge).m_type);
4860         terminate(Uncountable);
4861         return m_out.int64Zero;
4862     }
4863
4864     LValue lowInt52(Edge edge)
4865     {
4866         return lowInt52(edge, Int52);
4867     }
4868
4869     LValue lowStrictInt52(Edge edge)
4870     {
4871         return lowInt52(edge, StrictInt52);
4872     }
4873
4874     bool betterUseStrictInt52(Node* node)
4875     {
4876         return !isValid(m_int52Values.get(node));
4877     }
4878     bool betterUseStrictInt52(Edge edge)
4879     {
4880         return betterUseStrictInt52(edge.node());
4881     }
4882     template<typename T>
4883     Int52Kind bestInt52Kind(T node)
4884     {
4885         return betterUseStrictInt52(node) ? StrictInt52 : Int52;
4886     }
4887     Int52Kind opposite(Int52Kind kind)
4888     {
4889         switch (kind) {
4890         case Int52:
4891             return StrictInt52;
4892         case StrictInt52:
4893             return Int52;
4894         }
4895         LOWERING_FAILED(m_node, "Bad use kind");
4896         return Int52;
4897     }
4898
4899     LValue lowWhicheverInt52(Edge edge, Int52Kind& kind)
4900     {
4901         kind = bestInt52Kind(edge);
4902         return lowInt52(edge, kind);
4903     }
4904
4905     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4906     {
4907         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind()));
4908
4909         if (edge->op() == JSConstant) {
4910             JSValue value = m_graph.valueOfJSConstant(edge.node());
4911             if (!value.isCell()) {
4912                 terminate(Uncountable);
4913                 return m_out.intPtrZero;
4914             }
4915             return m_out.constIntPtr(value.asCell());
4916         }
4917
4918         LoweredNodeValue value = m_jsValueValues.get(edge.node());
4919         if (isValid(value)) {
4920             LValue uncheckedValue = value.value();
4921             FTL_TYPE_CHECK(
4922                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
4923             return uncheckedValue;
4924         }
4925
4926         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
4927         terminate(Uncountable);
4928         return m_out.intPtrZero;
4929     }
4930
4931     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4932     {
4933         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
4934
4935         LValue result = lowCell(edge, mode);
4936         speculateObject(edge, result);
4937         return result;
4938     }
4939
4940     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4941     {
4942         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse);
4943
4944         LValue result = lowCell(edge, mode);
4945         speculateString(edge, result);
4946         return result;
4947     }
4948
4949     LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4950     {
4951         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse);
4952
4953         LValue string = lowString(edge, mode);
4954         LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
4955         speculateStringIdent(edge, string, stringImpl);
4956         return stringImpl;
4957     }
4958
4959     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4960     {
4961         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
4962
4963         LValue result = lowCell(edge, mode);
4964         speculateNonNullObject(edge, result);
4965         return result;
4966     }
4967
4968     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4969     {
4970         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
4971
4972         if (edge->hasConstant()) {
4973             JSValue value = m_graph.valueOfJSConstant(edge.node());
4974             if (!value.isBoolean()) {
4975                 terminate(Uncountable);
4976                 return m_out.booleanFalse;
4977             }
4978             return m_out.constBool(value.asBoolean());
4979         }
4980
4981         LoweredNodeValue value = m_booleanValues.get(edge.node());
4982         if (isValid(value))
4983             return value.value();
4984
4985         value = m_jsValueValues.get(edge.node());
4986         if (isValid(value)) {
4987             LValue unboxedResult = value.value();
4988             FTL_TYPE_CHECK(
4989                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
4990             LValue result = unboxBoolean(unboxedResult);
4991             setBoolean(edge.node(), result);
4992             return result;
4993         }
4994
4995         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
4996         terminate(Uncountable);
4997         return m_out.booleanFalse;
4998     }
4999
5000     LValue lowDouble(Edge edge)
5001     {
5002         RELEASE_ASSERT(isDouble(edge.useKind()));
5003
5004         LoweredNodeValue value = m_doubleValues.get(edge.node());
5005         if (isValid(value))
5006             return value.value();
5007
5008         RELEASE_ASSERT(!m_state.forNode(edge).m_type);
5009         terminate(Uncountable);
5010         return m_out.doubleZero;
5011     }
5012
5013     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
5014     {
5015         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
5016         RELEASE_ASSERT(!isDouble(edge.useKind()));
5017         RELEASE_ASSERT(edge.useKind() != Int52RepUse);
5018
5019         if (edge->hasConstant())
5020             return m_out.constInt64(JSValue::encode(m_graph.valueOfJSConstant(edge.node())));
5021
5022         LoweredNodeValue value = m_jsValueValues.get(edge.node());
5023         if (isValid(value))
5024             return value.value();
5025
5026         value = m_int32Values.get(edge.node());
5027         if (isValid(value)) {
5028             LValue result = boxInt32(value.value());
5029             setJSValue(edge.node(), result);
5030             return result;
5031         }
5032
5033         value = m_booleanValues.get(edge.node());
5034         if (isValid(value)) {
5035             LValue result = boxBoolean(value.value());
5036             setJSValue(edge.node(), result);
5037             return result;
5038         }
5039
5040         LOWERING_FAILED(m_node, "Corrupt array class");
5041         return 0;
5042     }
5043
5044     LValue lowStorage(Edge edge)
5045     {
5046         LoweredNodeValue value = m_storageValues.get(edge.node());
5047         if (isValid(value))
5048             return value.value();
5049
5050         LValue result = lowCell(edge);
5051         setStorage(edge.node(), result);
5052         return result;
5053     }
5054
5055     LValue strictInt52ToInt32(Edge edge, LValue value)
5056     {
5057         LValue result = m_out.castToInt32(value);
5058         FTL_TYPE_CHECK(
5059             noValue(), edge, SpecInt32,
5060             m_out.notEqual(m_out.signExt(result, m_out.int64), value));
5061         setInt32(edge.node(), result);
5062         return result;
5063     }
5064
5065     LValue strictInt52ToDouble(LValue value)
5066     {
5067         return m_out.intToDouble(value);
5068     }
5069
5070     LValue strictInt52ToJSValue(LValue value)
5071     {
5072         LBasicBlock isInt32 = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isInt32 case"));
5073         LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isDouble case"));
5074         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue continuation"));
5075
5076         Vector<ValueFromBlock, 2> results;
5077
5078         LValue int32Value = m_out.castToInt32(value);
5079         m_out.branch(
5080             m_out.equal(m_out.signExt(int32Value, m_out.int64), value),
5081             unsure(isInt32), unsure(isDouble));
5082
5083         LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble);
5084
5085         results.append(m_out.anchor(boxInt32(int32Value)));
5086         m_out.jump(continuation);
5087
5088         m_out.appendTo(isDouble, continuation);
5089
5090         results.append(m_out.anchor(boxDouble(m_out.intToDouble(value))));
5091         m_out.jump(continuation);
5092
5093         m_out.appendTo(continuation, lastNext);
5094         return m_out.phi(m_out.int64, results);
5095     }
5096
5097     LValue strictInt52ToInt52(LValue value)
5098     {
5099         return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount));
5100     }
5101
5102     LValue int52ToStrictInt52(LValue value)
5103     {
5104         return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount));
5105     }
5106
5107     LValue isNotInt32(LValue jsValue)
5108     {
5109         return m_out.below(jsValue, m_tagTypeNumber);
5110     }
5111     LValue unboxInt32(LValue jsValue)
5112     {
5113         return m_out.castToInt32(jsValue);
5114     }
5115     LValue boxInt32(LValue value)
5116     {
5117         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
5118     }
5119
5120     LValue isCellOrMisc(LValue jsValue)
5121     {
5122         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
5123     }
5124     LValue isNotCellOrMisc(LValue jsValue)
5125     {
5126         return m_out.testNonZero64(jsValue, m_tagTypeNumber);
5127     }
5128
5129     LValue unboxDouble(LValue jsValue)
5130     {
5131         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
5132     }
5133     LValue boxDouble(LValue doubleValue)
5134     {
5135         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
5136     }
5137     LValue jsValueToDouble(Edge edge, LValue boxedValue)
5138     {
5139         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing int case"));
5140         LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing double case"));
5141         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing continuation"));
5142
5143         LValue isNotInt32;
5144         if (!m_interpreter.needsTypeCheck(edge, SpecInt32))
5145             isNotInt32 = m_out.booleanFalse;
5146         else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32))
5147             isNotInt32 = m_out.booleanTrue;
5148         else
5149             isNotInt32 = this->isNotInt32(boxedValue);
5150         m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
5151
5152         LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
5153
5154         ValueFromBlock intToDouble = m_out.anchor(
5155             m_out.intToDouble(unboxInt32(boxedValue)));
5156         m_out.jump(continuation);
5157
5158         m_out.appendTo(doubleCase, continuation);
5159
5160         FTL_TYPE_CHECK(
5161             jsValueValue(boxedValue), edge, SpecBytecodeNumber, isCellOrMisc(boxedValue));
5162
5163         ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedValue));
5164         m_out.jump(continuation);
5165
5166         m_out.appendTo(continuation, lastNext);
5167
5168         return m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
5169     }
5170
5171     LValue jsValueToStrictInt52(Edge edge, LValue boxedValue)
5172     {
5173         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing int case"));
5174         LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing double case"));
5175         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing continuation"));
5176
5177         LValue isNotInt32;
5178         if (!m_interpreter.needsTypeCheck(edge, SpecInt32))
5179             isNotInt32 = m_out.booleanFalse;
5180         else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32))
5181             isNotInt32 = m_out.booleanTrue;
5182         else
5183             isNotInt32 = this->isNotInt32(boxedValue);
5184         m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
5185
5186         LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
5187
5188         ValueFromBlock intToInt52 = m_out.anchor(
5189             m_out.signExt(unboxInt32(boxedValue), m_out.int64));
5190         m_out.jump(continuation);
5191
5192         m_out.appendTo(doubleCase, continuation);
5193
5194         LValue possibleResult = m_out.call(
5195             m_out.operation(operationConvertBoxedDoubleToInt52), boxedValue);
5196         FTL_TYPE_CHECK(
5197             jsValueValue(boxedValue), edge, SpecInt32 | SpecInt52AsDouble,
5198             m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
5199
5200         ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult);
5201         m_out.jump(continuation);
5202
5203         m_out.appendTo(continuation, lastNext);
5204
5205         return m_out.phi(m_out.int64, intToInt52, doubleToInt52);
5206     }
5207
5208     LValue doubleToStrictInt52(Edge edge, LValue value)
5209     {
5210         LValue possibleResult = m_out.call(
5211             m_out.operation(operationConvertDoubleToInt52), value);
5212         FTL_TYPE_CHECK(
5213             doubleValue(value), edge, SpecInt52AsDouble,
5214             m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
5215
5216         return possibleResult;
5217     }
5218
5219     LValue isNumber(LValue jsValue)
5220     {
5221         return isNotCellOrMisc(jsValue);
5222     }
5223     LValue isNotNumber(LValue jsValue)
5224     {
5225         return isCellOrMisc(jsValue);
5226     }
5227
5228     LValue isNotCell(LValue jsValue)
5229     {
5230         return m_out.testNonZero64(jsValue, m_tagMask);
5231     }
5232
5233     LValue isCell(LValue jsValue)
5234     {
5235         return m_out.testIsZero64(jsValue, m_tagMask);
5236     }
5237
5238     LValue isNotMisc(LValue value)
5239     {
5240         return m_out.above(value, m_out.constInt64(TagBitTypeOther | TagBitBool | TagBitUndefined));
5241     }
5242
5243     LValue isMisc(LValue value)
5244     {
5245         return m_out.bitNot(isNotMisc(value));
5246     }
5247
5248     LValue isNotBoolean(LValue jsValue)
5249     {
5250         return m_out.testNonZero64(
5251             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
5252             m_out.constInt64(~1));
5253     }
5254     LValue isBoolean(LValue jsValue)
5255     {
5256         return m_out.bitNot(isNotBoolean(jsValue));
5257     }
5258     LValue unboxBoolean(LValue jsValue)
5259     {
5260         // We want to use a cast that guarantees that LLVM knows that even the integer
5261         // value is just 0 or 1. But for now we do it the dumb way.
5262         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
5263     }
5264     LValue boxBoolean(LValue value)
5265     {
5266         return m_out.select(
5267             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
5268     }
5269
5270     LValue isNotOther(LValue value)
5271     {
5272         return m_out.notEqual(
5273             m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
5274             m_out.constInt64(ValueNull));
5275     }
5276     LValue isOther(LValue value)
5277     {
5278         return m_out.equal(
5279             m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
5280             m_out.constInt64(ValueNull));
5281     }
5282
5283     void speculate(Edge edge)
5284     {
5285         switch (edge.useKind()) {
5286         case UntypedUse:
5287             break;
5288         case KnownInt32Use:
5289         case KnownStringUse:
5290         case DoubleRepUse:
5291         case Int52RepUse:
5292             ASSERT(!m_interpreter.needsTypeCheck(edge));
5293             break;
5294         case Int32Use:
5295             speculateInt32(edge);
5296             break;
5297         case CellUse:
5298             speculateCell(edge);
5299             break;
5300         case KnownCellUse:
5301             ASSERT(!m_interpreter.needsTypeCheck(edge));
5302             break;
5303         case MachineIntUse:
5304             speculateMachineInt(edge);
5305             break;
5306         case ObjectUse:
5307             speculateObject(edge);
5308             break;
5309         case ObjectOrOtherUse:
5310             speculateObjectOrOther(edge);
5311             break;
5312         case FinalObjectUse:
5313             speculateFinalObject(edge);
5314             break;
5315         case StringUse:
5316             speculateString(edge);
5317             break;
5318         case StringIdentUse:
5319             speculateStringIdent(edge);
5320             break;
5321         case StringObjectUse:
5322             speculateStringObject(edge);
5323             break;
5324         case StringOrStringObjectUse:
5325             speculateStringOrStringObject(edge);
5326             break;
5327         case NumberUse:
5328             speculateNumber(edge);
5329             break;
5330         case DoubleRepRealUse:
5331             speculateDoubleReal(edge);
5332             break;
5333         case DoubleRepMachineIntUse:
5334             speculateDoubleRepMachineInt(edge);
5335             break;
5336         case BooleanUse:
5337             speculateBoolean(edge);
5338             break;
5339         case NotStringVarUse:
5340             speculateNotStringVar(edge);
5341             break;
5342         case NotCellUse:
5343             speculateNotCell(edge);
5344             break;
5345         case OtherUse:
5346             speculateOther(edge);
5347             break;
5348         case MiscUse:
5349             speculateMisc(edge);
5350             break;
5351         default:
5352             LOWERING_FAILED(m_node, "Unsupported speculation use kind");
5353             return;
5354         }
5355     }
5356
5357     void speculate(Node*, Edge edge)
5358     {
5359         speculate(edge);
5360     }
5361
5362     void speculateInt32(Edge edge)
5363     {
5364         lowInt32(edge);
5365     }
5366
5367     void speculateCell(Edge edge)
5368     {
5369         lowCell(edge);
5370     }
5371
5372     void speculateMachineInt(Edge edge)
5373     {
5374         if (!m_interpreter.needsTypeCheck(edge))
5375             return;
5376
5377         jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation));
5378     }
5379
5380     LValue isObject(LValue cell)
5381     {
5382         return m_out.notEqual(
5383             m_out.load32(cell, m_heaps.JSCell_structureID),
5384             m_out.constInt32(vm().stringStructure->id()));
5385     }
5386
5387     LValue isNotString(LValue cell)
5388     {
5389         return isObject(cell);
5390     }
5391
5392     LValue isString(LValue cell)
5393     {
5394         return m_out.equal(
5395             m_out.load32(cell, m_heaps.JSCell_structureID),
5396             m_out.constInt32(vm().stringStructure->id()));
5397     }
5398
5399     LValue isNotObject(LValue cell)
5400     {
5401         return isString(cell);
5402     }
5403
5404     LValue isArrayType(LValue cell, ArrayMode arrayMode)
5405     {
5406         switch (arrayMode.type()) {
5407         case Array::Int32:
5408         case Array::Double:
5409         case Array::Contiguous: {
5410             LValue indexingType = m_out.load8(cell, m_heaps.JSCell_indexingType);
5411
5412             switch (arrayMode.arrayClass()) {
5413             case Array::OriginalArray:
5414                 LOWERING_FAILED(m_node, "Unexpected original array");
5415                 return 0;
5416
5417             case Array::Array:
5418                 return m_out.equal(
5419                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
5420                     m_out.constInt8(IsArray | arrayMode.shapeMask()));
5421
5422             case Array::NonArray:
5423             case Array::OriginalNonArray:
5424                 return m_out.equal(
5425                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
5426                     m_out.constInt8(arrayMode.shapeMask()));
5427
5428             case Array::PossiblyArray:
5429                 return m_out.equal(
5430                     m_out.bitAnd(indexingType, m_out.constInt8(IndexingShapeMask)),
5431                     m_out.constInt8(arrayMode.shapeMask()));
5432             }
5433
5434             LOWERING_FAILED(m_node, "Corrupt array class");
5435             return 0;
5436         }
5437
5438         default:
5439             return m_out.equal(
5440                 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
5441                 m_out.constInt8(typeForTypedArrayType(arrayMode.typedArrayType())));
5442         }
5443     }
5444
5445     LValue isType(LValue cell, JSType type)
5446     {
5447         return m_out.equal(
5448             m_out.load8(cell, m_heaps.JSCell_typeInfoType),
5449             m_out.constInt8(type));
5450     }
5451
5452     LValue isNotType(LValue cell, JSType type)
5453     {
5454         return m_out.bitNot(isType(cell, type));
5455     }
5456
5457     void speculateObject(Edge edge, LValue cell)
5458     {
5459         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
5460     }
5461
5462     void speculateObject(Edge edge)
5463     {
5464         speculateObject(edge, lowCell(edge));
5465     }
5466
5467     void speculateObjectOrOther(Edge edge)
5468     {
5469         if (!m_interpreter.needsTypeCheck(edge))
5470             return;
5471
5472         LValue value = lowJSValue(edge);
5473
5474         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case"));
5475         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case"));
5476         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation"));
5477
5478         m_out.branch(isNotCell(value), unsure(primitiveCase), unsure(cellCase));
5479
5480         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
5481
5482         FTL_TYPE_CHECK(
5483             jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value));
5484
5485         m_out.jump(continuation);
5486
5487         m_out.appendTo(primitiveCase, continuation);
5488
5489         FTL_TYPE_CHECK(
5490             jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
5491
5492         m_out.jump(continuation);
5493
5494         m_out.appendTo(continuation, lastNext);
5495     }
5496
5497     void speculateFinalObject(Edge edge, LValue cell)
5498     {
5499         FTL_TYPE_CHECK(
5500             jsValueValue(cell), edge, SpecFinalObject, isNotType(cell, FinalObjectType));
5501     }
5502
5503     void speculateFinalObject(Edge edge)
5504     {
5505         speculateFinalObject(edge, lowCell(edge));
5506     }
5507
5508     void speculateString(Edge edge, LValue cell)
5509     {
5510         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString | ~SpecCell, isNotString(cell));
5511     }
5512
5513     void speculateString(Edge edge)
5514     {
5515         speculateString(edge, lowCell(edge));
5516     }
5517
5518     void speculateStringIdent(Edge edge, LValue string, LValue stringImpl)
5519     {
5520         if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString))
5521             return;
5522
5523         speculate(BadType, jsValueValue(string), edge.node(), m_out.isNull(stringImpl));
5524         speculate(
5525             BadType, jsValueValue(string), edge.node(),
5526             m_out.testIsZero32(
5527                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
5528                 m_out.constInt32(StringImpl::flagIsAtomic())));
5529         m_interpreter.filter(edge, SpecStringIdent | ~SpecString);
5530     }
5531
5532     void speculateStringIdent(Edge edge)
5533     {
5534         lowStringIdent(edge);
5535     }
5536
5537     void speculateStringObject(Edge edge)
5538     {
5539         if (!m_interpreter.needsTypeCheck(edge, SpecStringObject))
5540             return;
5541
5542         speculateStringObjectForCell(edge, lowCell(edge));
5543         m_interpreter.filter(edge, SpecStringObject);
5544     }
5545
5546     void speculateStringOrStringObject(Edge edge)
5547     {
5548         if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject))
5549             return;
5550
5551         LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject not string case"));
5552         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject continuation"));
5553
5554         LValue structureID = m_out.load32(lowCell(edge), m_heaps.JSCell_structureID);
5555         m_out.branch(
5556             m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
5557             unsure(continuation), unsure(notString));
5558
5559         LBasicBlock lastNext = m_out.appendTo(notString, continuation);
5560         speculateStringObjectForStructureID(edge, structureID);
5561         m_out.jump(continuation);
5562
5563         m_out.appendTo(continuation, lastNext);
5564
5565         m_interpreter.filter(edge, SpecString | SpecStringObject);
5566     }
5567
5568     void speculateStringObjectForCell(Edge edge, LValue cell)
5569     {
5570         speculateStringObjectForStructureID(edge, m_out.load32(cell, m_heaps.JSCell_structureID));
5571     }
5572
5573     void speculateStringObjectForStructureID(Edge edge, LValue structureID)
5574     {
5575         Structure* stringObjectStructure =
5576             m_graph.globalObjectFor(m_node->origin.semantic)->stringObjectStructure();
5577
5578         if (m_state.forNode(edge).m_currentKnownStructure.isSubsetOf(StructureSet(stringObjectStructure)))
5579             return;
5580
5581         speculate(
5582             NotStringObject, noValue(), 0,
5583             m_out.notEqual(structureID, weakStructure(stringObjectStructure)));
5584     }
5585
5586     void speculateNonNullObject(Edge edge, LValue cell)
5587     {
5588         FTL_TYPE_CHECK(
5589             jsValueValue(cell), edge, SpecObject,
5590             m_out.equal(
5591                 m_out.load32(cell, m_heaps.JSCell_structureID),
5592                 m_out.constInt32(vm().stringStructure->id())));
5593         if (masqueradesAsUndefinedWatchpointIsStillValid())
5594             return;
5595
5596         speculate(
5597             BadType, jsValueValue(cell), edge.node(),
5598             m_out.testNonZero8(
5599                 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
5600                 m_out.constInt8(MasqueradesAsUndefined)));
5601     }
5602
5603     void speculateNumber(Edge edge)
5604     {
5605         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5606         FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value));
5607     }
5608
5609     void speculateDoubleReal(Edge edge)
5610     {
5611         // Do an early return here because lowDouble() can create a lot of control flow.
5612         if (!m_interpreter.needsTypeCheck(edge))
5613             return;
5614
5615         LValue value = lowDouble(edge);
5616         FTL_TYPE_CHECK(
5617             doubleValue(value), edge, SpecDoubleReal,
5618             m_out.doubleNotEqualOrUnordered(value, value));
5619     }
5620
5621     void speculateDoubleRepMachineInt(Edge edge)
5622     {
5623         if (!m_interpreter.needsTypeCheck(edge))
5624             return;
5625
5626         doubleToStrictInt52(edge, lowDouble(edge));
5627     }
5628
5629     void speculateBoolean(Edge edge)
5630     {
5631         lowBoolean(edge);
5632     }
5633
5634     void speculateNotStringVar(Edge edge)
5635     {
5636         if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar))
5637             return;
5638
5639         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5640
5641         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is cell case"));
5642         LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is string case"));
5643         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar continuation"));
5644
5645         m_out.branch(isCell(value), unsure(isCellCase), unsure(continuation));
5646
5647         LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
5648         m_out.branch(isString(value), unsure(isStringCase), unsure(continuation));
5649
5650         m_out.appendTo(isStringCase, continuation);
5651         speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value));
5652         m_out.jump(continuation);
5653
5654         m_out.appendTo(continuation, lastNext);
5655     }
5656
5657     void speculateNotCell(Edge edge)
5658     {
5659         if (!m_interpreter.needsTypeCheck(edge))
5660             return;
5661
5662         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5663         typeCheck(jsValueValue(value), edge, ~SpecCell, isCell(value));
5664     }
5665
5666     void speculateOther(Edge edge)
5667     {
5668         if (!m_interpreter.needsTypeCheck(edge))
5669             return;
5670
5671         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5672         typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value));
5673     }
5674
5675     void speculateMisc(Edge edge)
5676     {
5677         if (!m_interpreter.needsTypeCheck(edge))
5678             return;
5679
5680         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5681         typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value));
5682     }
5683
5684     bool masqueradesAsUndefinedWatchpointIsStillValid()
5685     {
5686         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic);
5687     }
5688
5689     LValue loadMarkByte(LValue base)
5690     {
5691         return m_out.load8(base, m_heaps.JSCell_gcData);
5692     }
5693
5694     void emitStoreBarrier(LValue base)
5695     {
5696 #if ENABLE(GGC)
5697         LBasicBlock isMarkedAndNotRemembered = FTL_NEW_BLOCK(m_out, ("Store barrier is marked block"));
5698         LBasicBlock bufferHasSpace = FTL_NEW_BLOCK(m_out, ("Store barrier buffer has space"));
5699         LBasicBlock bufferIsFull = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full"));
5700         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation"));
5701
5702         // Check the mark byte.
5703         m_out.branch(
5704             m_out.notZero8(loadMarkByte(base)), usually(continuation), rarely(isMarkedAndNotRemembered));
5705
5706         // Append to the write barrier buffer.
5707         LBasicBlock lastNext = m_out.appendTo(isMarkedAndNotRemembered, bufferHasSpace);
5708         LValue currentBufferIndex = m_out.load32(m_out.absolute(&vm().heap.writeBarrierBuffer().m_currentIndex));
5709         LValue bufferCapacity = m_out.load32(m_out.absolute(&vm().heap.writeBarrierBuffer().m_capacity));
5710         m_out.branch(
5711             m_out.lessThan(currentBufferIndex, bufferCapacity),
5712             usually(bufferHasSpace), rarely(bufferIsFull));
5713
5714         // Buffer has space, store to it.
5715         m_out.appendTo(bufferHasSpace, bufferIsFull);
5716         LValue writeBarrierBufferBase = m_out.loadPtr(m_out.absolute(&vm().heap.writeBarrierBuffer().m_buffer));
5717         m_out.storePtr(base, m_out.baseIndex(m_heaps.WriteBarrierBuffer_bufferContents, writeBarrierBufferBase, m_out.zeroExt(currentBufferIndex, m_out.intPtr), ScalePtr));
5718         m_out.store32(m_out.add(currentBufferIndex, m_out.constInt32(1)), m_out.absolute(&vm().heap.writeBarrierBuffer().m_currentIndex));
5719         m_out.jump(continuation);
5720
5721         // Buffer is out of space, flush it.
5722         m_out.appendTo(bufferIsFull, continuation);
5723         vmCall(m_out.operation(operationFlushWriteBarrierBuffer), m_callFrame, base, NoExceptions);
5724         m_out.jump(continuation);
5725
5726         m_out.appendTo(continuation, lastNext);
5727 #else
5728         UNUSED_PARAM(base);
5729 #endif
5730     }
5731
5732     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
5733
5734     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
5735     {
5736         callPreflight();
5737         LValue result = m_out.call(function);
5738         callCheck(mode);
5739         return result;
5740     }
5741     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
5742     {
5743         callPreflight();
5744         LValue result = m_out.call(function, arg1);
5745         callCheck(mode);
5746         return result;
5747     }
5748     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
5749     {
5750         callPreflight();
5751         LValue result = m_out.call(function, arg1, arg2);
5752         callCheck(mode);
5753         return result;
5754     }
5755     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
5756     {
5757         callPreflight();
5758         LValue result = m_out.call(function, arg1, arg2, arg3);
5759         callCheck(mode);
5760         return result;
5761     }
5762     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
5763     {
5764         callPreflight();
5765         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
5766         callCheck(mode);
5767         return result;
5768     }
5769
5770     void callPreflight(CodeOrigin codeOrigin)
5771     {
5772         m_out.store32(
5773             m_out.constInt32(
5774                 CallFrame::Location::encodeAsCodeOriginIndex(
5775                     m_ftlState.jitCode->common.addCodeOrigin(codeOrigin))),
5776             tagFor(JSStack::ArgumentCount));
5777     }
5778     void callPreflight()
5779     {
5780         callPreflight(m_node->origin.semantic);
5781     }
5782
5783     void callCheck(ExceptionCheckMode mode = CheckExceptions)
5784     {
5785         if (mode == NoExceptions)
5786             return;
5787
5788         if (Options::enableExceptionFuzz())
5789             m_out.call(m_out.operation(operationExceptionFuzz));
5790
5791         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
5792
5793         m_out.branch(
5794             m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))),
5795             rarely(m_handleExceptions), usually(continuation));
5796
5797         m_out.appendTo(continuation);
5798     }
5799
5800     LBasicBlock lowBlock(BasicBlock* block)
5801     {
5802         return m_blocks.get(block);
5803     }
5804
5805     void initializeOSRExitStateForBlock()
5806     {
5807         m_availability = m_highBlock->ssa->availabilityAtHead;
5808     }
5809
5810     void appendOSRExit(
5811         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
5812     {
5813         if (verboseCompilationEnabled()) {
5814             dataLog("    OSR exit #", m_ftlState.jitCode->osrExit.size(), " with availability: ", m_availability, "\n");
5815             if (!m_availableRecoveries.isEmpty())
5816                 dataLog("        Available recoveries: ", listDump(m_availableRecoveries), "\n");
5817         }
5818
5819         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.finalizer->osrExit.size());
5820
5821         m_ftlState.jitCode->osrExit.append(OSRExit(
5822             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
5823             m_codeOriginForExitTarget, m_codeOriginForExitProfile,
5824             m_availability.numberOfArguments(), m_availability.numberOfLocals()));
5825         m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
5826
5827         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
5828
5829         LBasicBlock lastNext = 0;
5830         LBasicBlock continuation = 0;
5831
5832         LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
5833         continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
5834
5835         m_out.branch(failCondition, rarely(failCase), usually(continuation));
5836
5837         lastNext = m_out.appendTo(failCase, continuation);
5838
5839         emitOSRExitCall(exit, lowValue);
5840
5841         m_out.unreachable();
5842
5843         m_out.appendTo(continuation, lastNext);
5844     }
5845
5846     void emitOSRExitCall(OSRExit& exit, FormattedValue lowValue)
5847     {
5848         ExitArgumentList arguments;
5849
5850         CodeOrigin codeOrigin = exit.m_codeOrigin;
5851
5852         buildExitArguments(exit, arguments, lowValue, codeOrigin);
5853
5854         callStackmap(exit, arguments);
5855     }
5856
5857     void buildExitArguments(
5858         OSRExit& exit, ExitArgumentList& arguments, FormattedValue lowValue,
5859         CodeOrigin codeOrigin)
5860     {
5861         if (!!lowValue)
5862             arguments.append(lowValue.value());
5863
5864         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
5865             int operand = exit.m_values.operandForIndex(i);
5866             bool isLive = m_graph.isLiveInBytecode(VirtualRegister(operand), codeOrigin);
5867             if (!isLive) {
5868                 exit.m_values[i] = ExitValue::dead();
5869                 continue;
5870             }
5871
5872             Availability availability = m_availability[i];
5873             FlushedAt flush = availability.flushedAt();
5874             switch (flush.format()) {
5875             case DeadFlush:
5876             case ConflictingFlush:
5877                 if (availability.hasNode()) {
5878                     addExitArgumentForNode(exit, arguments, i, availability.node());
5879                     break;
5880                 }
5881
5882                 if (Options::validateFTLOSRExitLiveness()) {
5883                     dataLog("Expected r", operand, " to be available but it wasn't.\n");
5884                     RELEASE_ASSERT_NOT_REACHED();
5885                 }
5886
5887                 // This means that the DFG's DCE proved that the value is dead in bytecode
5888                 // even though the bytecode liveness analysis thinks it's live. This is
5889                 // acceptable since the DFG's DCE is by design more aggressive while still
5890                 // being sound.
5891                 exit.m_values[i] = ExitValue::dead();
5892                 break;
5893
5894             case FlushedJSValue:
5895             case FlushedCell:
5896             case FlushedBoolean:
5897                 exit.m_values[i] = ExitValue::inJSStack(flush.virtualRegister());
5898                 break;
5899
5900             case FlushedInt32:
5901                 exit.m_values[i] = ExitValue::inJSStackAsInt32(flush.virtualRegister());
5902                 break;
5903
5904             case FlushedInt52:
5905                 exit.m_values[i] = ExitValue::inJSStackAsInt52(flush.virtualRegister());
5906                 break;
5907
5908             case FlushedDouble:
5909                 exit.m_values[i] = ExitValue::inJSStackAsDouble(flush.virtualRegister());
5910                 break;
5911
5912             case FlushedArguments:
5913                 exit.m_values[i] = ExitValue::argumentsObjectThatWasNotCreated();
5914                 break;
5915             }
5916         }
5917
5918         if (verboseCompilationEnabled())
5919             dataLog("        Exit values: ", exit.m_values, "\n");
5920     }
5921
5922     void callStackmap(OSRExit& exit, ExitArgumentList& arguments)
5923     {
5924         exit.m_stackmapID = m_stackmapIDs++;
5925         arguments.insert(0, m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
5926         arguments.insert(0, m_out.constInt64(exit.m_stackmapID));
5927
5928         m_out.call(m_out.stackmapIntrinsic(), arguments);
5929     }
5930
5931     void addExitArgumentForNode(
5932         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
5933     {
5934         ASSERT(node->shouldGenerate());
5935         ASSERT(node->hasResult());
5936
5937         if (tryToSetConstantExitArgument(exit, index, node))
5938             return;
5939
5940         for (unsigned i = 0; i < m_availableRecoveries.size(); ++i) {
5941             AvailableRecovery recovery = m_availableRecoveries[i];
5942             if (recovery.node() != node)
5943                 continue;
5944
5945             exit.m_values[index] = ExitValue::recovery(
5946                 recovery.opcode(), arguments.size(), arguments.size() + 1,
5947                 recovery.format());
5948             arguments.append(recovery.left());
5949             arguments.append(recovery.right());
5950             return;
5951         }
5952
5953         LoweredNodeValue value = m_int32Values.get(node);
5954         if (isValid(value)) {
5955             addExitArgument(exit, arguments, index, ValueFormatInt32, value.value());
5956             return;
5957         }
5958
5959         value = m_int52Values.get(node);
5960         if (isValid(value)) {
5961             addExitArgument(exit, arguments, index, ValueFormatInt52, value.value());
5962             return;
5963         }
5964
5965         value = m_strictInt52Values.get(node);
5966         if (isValid(value)) {
5967             addExitArgument(exit, arguments, index, ValueFormatStrictInt52, value.value());
5968             return;
5969         }
5970
5971         value = m_booleanValues.get(node);
5972         if (isValid(value)) {
5973             LValue valueToPass = m_out.zeroExt(value.value(), m_out.int32);
5974             addExitArgument(exit, arguments, index, ValueFormatBoolean, valueToPass);
5975             return;
5976         }
5977
5978         value = m_jsValueValues.get(node);
5979         if (isValid(value)) {
5980             addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value());
5981             return;
5982         }
5983
5984         value = m_doubleValues.get(node);
5985         if (isValid(value)) {
5986             addExitArgument(exit, arguments, index, ValueFormatDouble, value.value());
5987             return;
5988         }
5989
5990         startCrashing();
5991         dataLog("Cannot find value for node: ", node, " while compiling exit at ", exit.m_codeOrigin, " in node ", m_node, "\n");
5992         m_graph.dump();
5993         RELEASE_ASSERT_NOT_REACHED();
5994     }
5995
5996     bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node)
5997     {
5998         if (!node)
5999             return false;
6000
6001         switch (node->op()) {
6002         case JSConstant:
6003         case Int52Constant:
6004         case DoubleConstant:
6005         case WeakJSConstant:
6006             exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node));
6007             return true;
6008         case PhantomArguments:
6009             exit.m_values[index] = ExitValue::argumentsObjectThatWasNotCreated();
6010             return true;
6011         default:
6012             return false;
6013         }
6014     }
6015
6016     void addExitArgument(
6017         OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format,
6018         LValue value)
6019     {
6020         exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
6021         arguments.append(value);
6022     }
6023
6024     bool doesKill(Edge edge)
6025     {
6026         if (edge.doesNotKill())
6027             return false;
6028
6029         if (edge->hasConstant())
6030             return false;
6031
6032         return true;
6033     }
6034
6035     void addAvailableRecovery(
6036         Node* node, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
6037     {
6038         m_availableRecoveries.append(AvailableRecovery(node, opcode, left, right, format));
6039     }
6040
6041     void addAvailableRecovery(
6042         Edge edge, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
6043     {
6044         addAvailableRecovery(edge.node(), opcode, left, right, format);
6045     }
6046
6047     void setInt32(Node* node, LValue value)
6048     {
6049         m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
6050     }
6051     void setInt52(Node* node, LValue value)
6052     {
6053         m_int52Values.set(node, LoweredNodeValue(value, m_highBlock));
6054     }
6055     void setStrictInt52(Node* node, LValue value)
6056     {
6057         m_strictInt52Values.set(node, LoweredNodeValue(value, m_highBlock));
6058     }
6059     void setInt52(Node* node, LValue value, Int52Kind kind)
6060     {
6061         switch (kind) {
6062         case Int52:
6063             setInt52(node, value);
6064             return;
6065
6066         case StrictInt52:
6067             setStrictInt52(node, value);
6068             return;
6069         }
6070
6071         LOWERING_FAILED(m_node, "Corrupt int52 kind");
6072     }
6073     void setJSValue(Node* node, LValue value)
6074     {
6075         m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
6076     }
6077     void setBoolean(Node* node, LValue value)
6078     {
6079         m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
6080     }
6081     void setStorage(Node* node, LValue value)
6082     {
6083         m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
6084     }
6085     void setDouble(Node* node, LValue value)
6086     {
6087         m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
6088     }
6089
6090     void setInt32(LValue value)
6091     {
6092         setInt32(m_node, value);
6093     }
6094     void setInt52(LValue value)
6095     {
6096         setInt52(m_node, value);
6097     }
6098     void setStrictInt52(LValue value)
6099     {
6100         setStrictInt52(m_node, value);
6101     }
6102     void setInt52(LValue value, Int52Kind kind)
6103     {
6104         setInt52(m_node, value, kind);
6105     }
6106     void setJSValue(LValue value)
6107     {
6108         setJSValue(m_node, value);
6109     }
6110     void setBoolean(LValue value)
6111     {
6112         setBoolean(m_node, value);
6113     }
6114     void setStorage(LValue value)
6115     {
6116         setStorage(m_node, value);
6117     }
6118     void setDouble(LValue value)
6119     {
6120         setDouble(m_node, value);
6121     }
6122
6123     bool isValid(const LoweredNodeValue& value)
6124     {
6125         if (!value)
6126             return false;
6127         if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
6128             return false;
6129         return true;
6130     }
6131
6132     void addWeakReference(JSCell* target)
6133     {
6134         m_graph.m_plan.weakReferences.addLazily(target);
6135     }
6136
6137     LValue loadStructure(LValue value)
6138     {
6139         LValue tableIndex = m_out.load32(value, m_heaps.JSCell_structureID);
6140         LValue tableBase = m_out.loadPtr(
6141             m_out.absolute(vm().heap.structureIDTable().base()));
6142         LValue pointerIntoTable = m_out.baseIndex(
6143             tableBase, m_out.zeroExt(tableIndex, m_out.intPtr), ScaleEight);
6144         return m_out.loadPtr(TypedPointer(m_heaps.structureTable, pointerIntoTable));
6145     }
6146
6147     LValue weakPointer(JSCell* pointer)
6148     {
6149         addWeakReference(pointer);
6150         return m_out.constIntPtr(pointer);
6151     }
6152
6153     LValue weakStructure(Structure* structure)
6154     {
6155         addWeakReference(structure);
6156         return m_out.constInt32(structure->id());
6157     }
6158
6159     TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
6160     {
6161         return m_out.address(base, m_heaps.variables[operand], offset);
6162     }
6163     TypedPointer payloadFor(LValue base, int operand)
6164     {
6165         return addressFor(base, operand, PayloadOffset);
6166     }
6167     TypedPointer tagFor(LValue base, int operand)
6168     {
6169         return addressFor(base, operand, TagOffset);
6170     }
6171     TypedPointer addressFor(int operand, ptrdiff_t offset = 0)
6172     {
6173         return addressFor(VirtualRegister(operand), offset);
6174     }
6175     TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0)
6176     {
6177         if (operand.isLocal())
6178             return addressFor(m_captured, operand.offset(), offset);
6179         return addressFor(m_callFrame, operand.offset(), offset);
6180     }
6181     TypedPointer payloadFor(int operand)
6182     {
6183         return payloadFor(VirtualRegister(operand));
6184     }
6185     TypedPointer payloadFor(VirtualRegister operand)
6186     {
6187         return addressFor(operand, PayloadOffset);
6188     }
6189     TypedPointer tagFor(int operand)
6190     {
6191         return tagFor(VirtualRegister(operand));
6192     }
6193     TypedPointer tagFor(VirtualRegister operand)
6194     {
6195         return addressFor(operand, TagOffset);
6196     }
6197
6198     NO_RETURN_DUE_TO_ASSERT void loweringFailed(Node* node, const char* file, int line, const char* function, const char* assertion)
6199     {
6200         if (!ASSERT_DISABLED) {
6201             dataLog("FTL ASSERTION FAILED: ", assertion, "\n");
6202             dataLog(file, "(", line, ") : ", function, "\n");
6203             dataLog("While handling node ", node, "\n");
6204             RELEASE_ASSERT_NOT_REACHED();
6205         }
6206
6207         m_loweringSucceeded = false;
6208     }
6209
6210     VM& vm() { return m_graph.m_vm; }
6211     CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
6212
6213     Graph& m_graph;
6214     State& m_ftlState;
6215     bool m_loweringSucceeded;
6216     AbstractHeapRepository m_heaps;
6217     Output m_out;
6218
6219     LBasicBlock m_prologue;
6220     LBasicBlock m_handleExceptions;
6221     HashMap<BasicBlock*, LBasicBlock> m_blocks;
6222
6223     LValue m_callFrame;
6224     LValue m_captured;
6225     LValue m_tagTypeNumber;
6226     LValue m_tagMask;
6227
6228     HashMap<Node*, LoweredNodeValue> m_int32Values;
6229     HashMap<Node*, LoweredNodeValue> m_strictInt52Values;
6230     HashMap<Node*, LoweredNodeValue> m_int52Values;
6231     HashMap<Node*, LoweredNodeValue> m_jsValueValues;
6232     HashMap<Node*, LoweredNodeValue> m_booleanValues;
6233     HashMap<Node*, LoweredNodeValue> m_storageValues;
6234     HashMap<Node*, LoweredNodeValue> m_doubleValues;
6235
6236     HashMap<Node*, LValue> m_phis;
6237
6238     Operands<Availability> m_availability;
6239
6240     Vector<AvailableRecovery, 3> m_availableRecoveries;
6241
6242     InPlaceAbstractState m_state;
6243     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
6244     BasicBlock* m_highBlock;
6245     BasicBlock* m_nextHighBlock;
6246     LBasicBlock m_nextLowBlock;
6247
6248     CodeOrigin m_codeOriginForExitTarget;
6249     CodeOrigin m_codeOriginForExitProfile;
6250     unsigned m_nodeIndex;
6251     Node* m_node;
6252
6253     uint32_t m_stackmapIDs;
6254 };
6255
6256 bool lowerDFGToLLVM(State& state)
6257 {
6258     LowerDFGToLLVM lowering(state);
6259     return lowering.lower();
6260 }
6261
6262 } } // namespace JSC::FTL
6263
6264 #endif // ENABLE(FTL_JIT)
6265