ftl/FTLLowerDFGToLLVM.cpp

   1 /*
   2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToLLVM.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "CodeBlockWithJITType.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "FTLAbstractHeapRepository.h"
  35 #include "FTLAvailableRecovery.h"
  36 #include "FTLForOSREntryJITCode.h"
  37 #include "FTLFormattedValue.h"
  38 #include "FTLInlineCacheSize.h"
  39 #include "FTLLoweredNodeValue.h"
  40 #include "FTLOutput.h"
  41 #include "FTLThunks.h"
  42 #include "FTLWeightedTarget.h"
  43 #include "OperandsInlines.h"
  44 #include "JSCInlines.h"
  45 #include "VirtualRegister.h"
  46 #include <atomic>
  47 #include <wtf/ProcessID.h>
  48
  49 namespace JSC { namespace FTL {
  50
  51 using namespace DFG;
  52
  53 static std::atomic<int> compileCounter;
  54
  55 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
  56 // significantly less dead code.
  57 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
  58         FormattedValue _ftc_lowValue = (lowValue);                      \
  59         Edge _ftc_highValue = (highValue);                              \
  60         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
  61         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
  62             break;                                                      \
  63         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
  64     } while (false)
  65
  66 class LowerDFGToLLVM {
  67 public:
  68     LowerDFGToLLVM(State& state)
  69         : m_graph(state.graph)
  70         , m_ftlState(state)
  71         , m_heaps(state.context)
  72         , m_out(state.context)
  73         , m_availability(OperandsLike, state.graph.block(0)->variablesAtHead)
  74         , m_state(state.graph)
  75         , m_interpreter(state.graph, m_state)
  76         , m_stackmapIDs(0)
  77     {
  78     }
  79
  80     void lower()
  81     {
  82         CString name;
  83         if (verboseCompilationEnabled()) {
  84             name = toCString(
  85                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
  86                 "_", codeBlock()->hash());
  87         } else
  88             name = "jsBody";
  89
  90         m_graph.m_dominators.computeIfNecessary(m_graph);
  91
  92         m_ftlState.module =
  93             llvm->ModuleCreateWithNameInContext(name.data(), m_ftlState.context);
  94
  95         m_ftlState.function = addFunction(
  96             m_ftlState.module, name.data(), functionType(m_out.int64));
  97         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
  98         if (isX86() && Options::llvmDisallowAVX()) {
  99             // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x
 100             // slower. It should be disabled.
 101             addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");
 102         }
 103
 104         if (verboseCompilationEnabled())
 105             dataLog("Function ready, beginning lowering.\n");
 106
 107         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
 108
 109         m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));
 110         LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));
 111         m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));
 112
 113         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 114             m_highBlock = m_graph.block(blockIndex);
 115             if (!m_highBlock)
 116                 continue;
 117             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
 118         }
 119
 120         m_out.appendTo(m_prologue, stackOverflow);
 121         createPhiVariables();
 122         LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));
 123         m_captured = m_out.add(
 124             m_out.ptrToInt(capturedAlloca, m_out.intPtr),
 125             m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));
 126
 127         // We should not create any alloca's after this point, since they will cease to
 128         // be mem2reg candidates.
 129
 130         m_ftlState.capturedStackmapID = m_stackmapIDs++;
 131         m_out.call(
 132             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),
 133             m_out.int32Zero, capturedAlloca);
 134
 135         m_callFrame = m_out.ptrToInt(
 136             m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);
 137         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 138         m_tagMask = m_out.constInt64(TagMask);
 139
 140         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
 141
 142         m_out.branch(
 143             didOverflowStack(), rarely(stackOverflow), usually(lowBlock(m_graph.block(0))));
 144
 145         m_out.appendTo(stackOverflow, m_handleExceptions);
 146         m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
 147         m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;
 148         m_out.call(
 149             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),
 150             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 151         m_out.unreachable();
 152
 153         m_out.appendTo(m_handleExceptions, lowBlock(m_graph.block(0)));
 154         m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;
 155         m_out.call(
 156             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),
 157             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 158         m_out.unreachable();
 159
 160         Vector<BasicBlock*> depthFirst;
 161         m_graph.getBlocksInDepthFirstOrder(depthFirst);
 162         for (unsigned i = 0; i < depthFirst.size(); ++i)
 163             compileBlock(depthFirst[i]);
 164
 165         if (Options::dumpLLVMIR())
 166             dumpModule(m_ftlState.module);
 167
 168         if (verboseCompilationEnabled())
 169             m_ftlState.dumpState("after lowering");
 170         if (validationEnabled())
 171             verifyModule(m_ftlState.module);
 172     }
 173
 174 private:
 175
 176     void createPhiVariables()
 177     {
 178         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 179             BasicBlock* block = m_graph.block(blockIndex);
 180             if (!block)
 181                 continue;
 182             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 183                 Node* node = block->at(nodeIndex);
 184                 if (node->op() != Phi)
 185                     continue;
 186                 LType type;
 187                 switch (node->flags() & NodeResultMask) {
 188                 case NodeResultDouble:
 189                     type = m_out.doubleType;
 190                     break;
 191                 case NodeResultInt32:
 192                     type = m_out.int32;
 193                     break;
 194                 case NodeResultInt52:
 195                     type = m_out.int64;
 196                     break;
 197                 case NodeResultBoolean:
 198                     type = m_out.boolean;
 199                     break;
 200                 case NodeResultJS:
 201                     type = m_out.int64;
 202                     break;
 203                 default:
 204                     RELEASE_ASSERT_NOT_REACHED();
 205                     break;
 206                 }
 207                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
 208             }
 209         }
 210     }
 211
 212     void compileBlock(BasicBlock* block)
 213     {
 214         if (!block)
 215             return;
 216
 217         if (verboseCompilationEnabled())
 218             dataLog("Compiling block ", *block, "\n");
 219
 220         m_highBlock = block;
 221
 222         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 223
 224         m_nextHighBlock = 0;
 225         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 226             m_nextHighBlock = m_graph.block(nextBlockIndex);
 227             if (m_nextHighBlock)
 228                 break;
 229         }
 230         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 231
 232         // All of this effort to find the next block gives us the ability to keep the
 233         // generated IR in roughly program order. This ought not affect the performance
 234         // of the generated code (since we expect LLVM to reorder things) but it will
 235         // make IR dumps easier to read.
 236         m_out.appendTo(lowBlock, m_nextLowBlock);
 237
 238         if (Options::ftlCrashes())
 239             m_out.crashNonTerminal();
 240
 241         if (!m_highBlock->cfaHasVisited) {
 242             m_out.crash();
 243             return;
 244         }
 245
 246         initializeOSRExitStateForBlock();
 247
 248         m_state.reset();
 249         m_state.beginBasicBlock(m_highBlock);
 250
 251         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 252             if (!compileNode(m_nodeIndex))
 253                 break;
 254         }
 255     }
 256
 257     bool compileNode(unsigned nodeIndex)
 258     {
 259         if (!m_state.isValid()) {
 260             m_out.unreachable();
 261             return false;
 262         }
 263
 264         m_node = m_highBlock->at(nodeIndex);
 265         m_codeOriginForExitProfile = m_node->origin.semantic;
 266         m_codeOriginForExitTarget = m_node->origin.forExit;
 267
 268         if (verboseCompilationEnabled())
 269             dataLog("Lowering ", m_node, "\n");
 270
 271         m_availableRecoveries.resize(0);
 272
 273         bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
 274
 275         switch (m_node->op()) {
 276         case Upsilon:
 277             compileUpsilon();
 278             break;
 279         case Phi:
 280             compilePhi();
 281             break;
 282         case JSConstant:
 283             break;
 284         case DoubleConstant:
 285             compileDoubleConstant();
 286             break;
 287         case Int52Constant:
 288             compileInt52Constant();
 289             break;
 290         case WeakJSConstant:
 291             compileWeakJSConstant();
 292             break;
 293         case PhantomArguments:
 294             compilePhantomArguments();
 295             break;
 296         case DoubleRep:
 297             compileDoubleRep();
 298             break;
 299         case ValueRep:
 300             compileValueRep();
 301             break;
 302         case Int52Rep:
 303             compileInt52Rep();
 304             break;
 305         case ValueToInt32:
 306             compileValueToInt32();
 307             break;
 308         case BooleanToNumber:
 309             compileBooleanToNumber();
 310             break;
 311         case GetArgument:
 312             compileGetArgument();
 313             break;
 314         case ExtractOSREntryLocal:
 315             compileExtractOSREntryLocal();
 316             break;
 317         case GetLocal:
 318             compileGetLocal();
 319             break;
 320         case SetLocal:
 321             compileSetLocal();
 322             break;
 323         case MovHint:
 324             compileMovHint();
 325             break;
 326         case GetMyArgumentsLength:
 327             compileGetMyArgumentsLength();
 328             break;
 329         case GetMyArgumentByVal:
 330             compileGetMyArgumentByVal();
 331             break;
 332         case ZombieHint:
 333             compileZombieHint();
 334             break;
 335         case Phantom:
 336         case HardPhantom:
 337             compilePhantom();
 338             break;
 339         case ToThis:
 340             compileToThis();
 341             break;
 342         case ValueAdd:
 343             compileValueAdd();
 344             break;
 345         case ArithAdd:
 346         case ArithSub:
 347             compileArithAddOrSub();
 348             break;
 349         case ArithMul:
 350             compileArithMul();
 351             break;
 352         case ArithDiv:
 353             compileArithDiv();
 354             break;
 355         case ArithMod:
 356             compileArithMod();
 357             break;
 358         case ArithMin:
 359         case ArithMax:
 360             compileArithMinOrMax();
 361             break;
 362         case ArithAbs:
 363             compileArithAbs();
 364             break;
 365         case ArithSin:
 366             compileArithSin();
 367             break;
 368         case ArithCos:
 369             compileArithCos();
 370             break;
 371         case ArithSqrt:
 372             compileArithSqrt();
 373             break;
 374         case ArithFRound:
 375             compileArithFRound();
 376             break;
 377         case ArithNegate:
 378             compileArithNegate();
 379             break;
 380         case BitAnd:
 381             compileBitAnd();
 382             break;
 383         case BitOr:
 384             compileBitOr();
 385             break;
 386         case BitXor:
 387             compileBitXor();
 388             break;
 389         case BitRShift:
 390             compileBitRShift();
 391             break;
 392         case BitLShift:
 393             compileBitLShift();
 394             break;
 395         case BitURShift:
 396             compileBitURShift();
 397             break;
 398         case UInt32ToNumber:
 399             compileUInt32ToNumber();
 400             break;
 401         case CheckStructure:
 402             compileCheckStructure();
 403             break;
 404         case StructureTransitionWatchpoint:
 405             compileStructureTransitionWatchpoint();
 406             break;
 407         case CheckFunction:
 408             compileCheckFunction();
 409             break;
 410         case CheckExecutable:
 411             compileCheckExecutable();
 412             break;
 413         case ArrayifyToStructure:
 414             compileArrayifyToStructure();
 415             break;
 416         case PutStructure:
 417             compilePutStructure();
 418             break;
 419         case PhantomPutStructure:
 420             compilePhantomPutStructure();
 421             break;
 422         case GetById:
 423             compileGetById();
 424             break;
 425         case PutByIdDirect:
 426         case PutById:
 427             compilePutById();
 428             break;
 429         case GetButterfly:
 430             compileGetButterfly();
 431             break;
 432         case ConstantStoragePointer:
 433             compileConstantStoragePointer();
 434             break;
 435         case GetIndexedPropertyStorage:
 436             compileGetIndexedPropertyStorage();
 437             break;
 438         case CheckArray:
 439             compileCheckArray();
 440             break;
 441         case GetArrayLength:
 442             compileGetArrayLength();
 443             break;
 444         case CheckInBounds:
 445             compileCheckInBounds();
 446             break;
 447         case GetByVal:
 448             compileGetByVal();
 449             break;
 450         case PutByVal:
 451         case PutByValAlias:
 452         case PutByValDirect:
 453             compilePutByVal();
 454             break;
 455         case ArrayPush:
 456             compileArrayPush();
 457             break;
 458         case ArrayPop:
 459             compileArrayPop();
 460             break;
 461         case NewObject:
 462             compileNewObject();
 463             break;
 464         case NewArray:
 465             compileNewArray();
 466             break;
 467         case NewArrayBuffer:
 468             compileNewArrayBuffer();
 469             break;
 470         case NewArrayWithSize:
 471             compileNewArrayWithSize();
 472             break;
 473         case GetTypedArrayByteOffset:
 474             compileGetTypedArrayByteOffset();
 475             break;
 476         case AllocatePropertyStorage:
 477             compileAllocatePropertyStorage();
 478             break;
 479         case ReallocatePropertyStorage:
 480             compileReallocatePropertyStorage();
 481             break;
 482         case ToString:
 483             compileToString();
 484             break;
 485         case ToPrimitive:
 486             compileToPrimitive();
 487             break;
 488         case MakeRope:
 489             compileMakeRope();
 490             break;
 491         case StringCharAt:
 492             compileStringCharAt();
 493             break;
 494         case StringCharCodeAt:
 495             compileStringCharCodeAt();
 496             break;
 497         case GetByOffset:
 498             compileGetByOffset();
 499             break;
 500         case MultiGetByOffset:
 501             compileMultiGetByOffset();
 502             break;
 503         case PutByOffset:
 504             compilePutByOffset();
 505             break;
 506         case MultiPutByOffset:
 507             compileMultiPutByOffset();
 508             break;
 509         case GetGlobalVar:
 510             compileGetGlobalVar();
 511             break;
 512         case PutGlobalVar:
 513             compilePutGlobalVar();
 514             break;
 515         case NotifyWrite:
 516             compileNotifyWrite();
 517             break;
 518         case GetCallee:
 519             compileGetCallee();
 520             break;
 521         case GetScope:
 522             compileGetScope();
 523             break;
 524         case GetMyScope:
 525             compileGetMyScope();
 526             break;
 527         case SkipScope:
 528             compileSkipScope();
 529             break;
 530         case GetClosureRegisters:
 531             compileGetClosureRegisters();
 532             break;
 533         case GetClosureVar:
 534             compileGetClosureVar();
 535             break;
 536         case PutClosureVar:
 537             compilePutClosureVar();
 538             break;
 539         case CompareEq:
 540             compileCompareEq();
 541             break;
 542         case CompareEqConstant:
 543             compileCompareEqConstant();
 544             break;
 545         case CompareStrictEq:
 546             compileCompareStrictEq();
 547             break;
 548         case CompareLess:
 549             compileCompareLess();
 550             break;
 551         case CompareLessEq:
 552             compileCompareLessEq();
 553             break;
 554         case CompareGreater:
 555             compileCompareGreater();
 556             break;
 557         case CompareGreaterEq:
 558             compileCompareGreaterEq();
 559             break;
 560         case LogicalNot:
 561             compileLogicalNot();
 562             break;
 563         case Call:
 564         case Construct:
 565             compileCallOrConstruct();
 566             break;
 567         case Jump:
 568             compileJump();
 569             break;
 570         case Branch:
 571             compileBranch();
 572             break;
 573         case Switch:
 574             compileSwitch();
 575             break;
 576         case Return:
 577             compileReturn();
 578             break;
 579         case ForceOSRExit:
 580             compileForceOSRExit();
 581             break;
 582         case Throw:
 583         case ThrowReferenceError:
 584             compileThrow();
 585             break;
 586         case InvalidationPoint:
 587             compileInvalidationPoint();
 588             break;
 589         case CheckArgumentsNotCreated:
 590             compileCheckArgumentsNotCreated();
 591             break;
 592         case IsUndefined:
 593             compileIsUndefined();
 594             break;
 595         case IsBoolean:
 596             compileIsBoolean();
 597             break;
 598         case IsNumber:
 599             compileIsNumber();
 600             break;
 601         case IsString:
 602             compileIsString();
 603             break;
 604         case IsObject:
 605             compileIsObject();
 606             break;
 607         case IsFunction:
 608             compileIsFunction();
 609             break;
 610         case CheckHasInstance:
 611             compileCheckHasInstance();
 612             break;
 613         case InstanceOf:
 614             compileInstanceOf();
 615             break;
 616         case CountExecution:
 617             compileCountExecution();
 618             break;
 619         case StoreBarrier:
 620             compileStoreBarrier();
 621             break;
 622         case StoreBarrierWithNullCheck:
 623             compileStoreBarrierWithNullCheck();
 624             break;
 625         case PhantomLocal:
 626         case SetArgument:
 627         case LoopHint:
 628         case VariableWatchpoint:
 629         case FunctionReentryWatchpoint:
 630         case TypedArrayWatchpoint:
 631         case AllocationProfileWatchpoint:
 632             break;
 633         default:
 634             dataLog("Unrecognized node in FTL backend:\n");
 635             m_graph.dump(WTF::dataFile(), "    ", m_node);
 636             dataLog("\n");
 637             dataLog("Full graph dump:\n");
 638             m_graph.dump();
 639             RELEASE_ASSERT_NOT_REACHED();
 640             break;
 641         }
 642
 643         if (shouldExecuteEffects)
 644             m_interpreter.executeEffects(nodeIndex);
 645
 646         return true;
 647     }
 648
 649     void compileUpsilon()
 650     {
 651         LValue destination = m_phis.get(m_node->phi());
 652
 653         switch (m_node->child1().useKind()) {
 654         case DoubleRepUse:
 655             m_out.set(lowDouble(m_node->child1()), destination);
 656             break;
 657         case Int32Use:
 658             m_out.set(lowInt32(m_node->child1()), destination);
 659             break;
 660         case Int52RepUse:
 661             m_out.set(lowInt52(m_node->child1()), destination);
 662             break;
 663         case BooleanUse:
 664             m_out.set(lowBoolean(m_node->child1()), destination);
 665             break;
 666         case CellUse:
 667             m_out.set(lowCell(m_node->child1()), destination);
 668             break;
 669         case UntypedUse:
 670             m_out.set(lowJSValue(m_node->child1()), destination);
 671             break;
 672         default:
 673             RELEASE_ASSERT_NOT_REACHED();
 674             break;
 675         }
 676     }
 677
 678     void compilePhi()
 679     {
 680         LValue source = m_phis.get(m_node);
 681
 682         switch (m_node->flags() & NodeResultMask) {
 683         case NodeResultDouble:
 684             setDouble(m_out.get(source));
 685             break;
 686         case NodeResultInt32:
 687             setInt32(m_out.get(source));
 688             break;
 689         case NodeResultInt52:
 690             setInt52(m_out.get(source));
 691             break;
 692         case NodeResultBoolean:
 693             setBoolean(m_out.get(source));
 694             break;
 695         case NodeResultJS:
 696             setJSValue(m_out.get(source));
 697             break;
 698         default:
 699             RELEASE_ASSERT_NOT_REACHED();
 700             break;
 701         }
 702     }
 703
 704     void compileDoubleConstant()
 705     {
 706         setDouble(m_out.constDouble(m_graph.valueOfNumberConstant(m_node)));
 707     }
 708
 709     void compileInt52Constant()
 710     {
 711         int64_t value = m_graph.valueOfJSConstant(m_node).asMachineInt();
 712
 713         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
 714         setStrictInt52(m_out.constInt64(value));
 715     }
 716
 717     void compileWeakJSConstant()
 718     {
 719         setJSValue(weakPointer(m_node->weakConstant()));
 720     }
 721
 722     void compilePhantomArguments()
 723     {
 724         setJSValue(m_out.constInt64(JSValue::encode(JSValue())));
 725     }
 726
 727     void compileDoubleRep()
 728     {
 729         switch (m_node->child1().useKind()) {
 730         case NumberUse: {
 731             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
 732             setDouble(jsValueToDouble(m_node->child1(), value));
 733             return;
 734         }
 735
 736         case Int52RepUse: {
 737             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
 738             return;
 739         }
 740
 741         default:
 742             RELEASE_ASSERT_NOT_REACHED();
 743         }
 744     }
 745
 746     void compileValueRep()
 747     {
 748         switch (m_node->child1().useKind()) {
 749         case DoubleRepUse: {
 750             LValue value = lowDouble(m_node->child1());
 751
 752             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
 753                 value = m_out.select(
 754                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
 755             }
 756
 757             setJSValue(boxDouble(value));
 758             return;
 759         }
 760
 761         case Int52RepUse: {
 762             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
 763             return;
 764         }
 765
 766         default:
 767             RELEASE_ASSERT_NOT_REACHED();
 768         }
 769     }
 770
 771     void compileInt52Rep()
 772     {
 773         switch (m_node->child1().useKind()) {
 774         case Int32Use:
 775             setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64));
 776             return;
 777
 778         case MachineIntUse:
 779             setStrictInt52(
 780                 jsValueToStrictInt52(
 781                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
 782             return;
 783
 784         case DoubleRepMachineIntUse:
 785             setStrictInt52(
 786                 doubleToStrictInt52(
 787                     m_node->child1(), lowDouble(m_node->child1())));
 788             return;
 789
 790         default:
 791             RELEASE_ASSERT_NOT_REACHED();
 792         }
 793     }
 794
 795     void compileValueToInt32()
 796     {
 797         switch (m_node->child1().useKind()) {
 798         case Int52RepUse:
 799             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
 800             break;
 801
 802         case DoubleRepUse:
 803             setInt32(doubleToInt32(lowDouble(m_node->child1())));
 804             break;
 805
 806         case NumberUse:
 807         case NotCellUse: {
 808             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
 809             if (isValid(value)) {
 810                 setInt32(value.value());
 811                 break;
 812             }
 813
 814             value = m_jsValueValues.get(m_node->child1().node());
 815             if (isValid(value)) {
 816                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
 817                 break;
 818             }
 819
 820             // We'll basically just get here for constants. But it's good to have this
 821             // catch-all since we often add new representations into the mix.
 822             setInt32(
 823                 numberOrNotCellToInt32(
 824                     m_node->child1(),
 825                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
 826             break;
 827         }
 828
 829         default:
 830             RELEASE_ASSERT_NOT_REACHED();
 831             break;
 832         }
 833     }
 834
 835     void compileBooleanToNumber()
 836     {
 837         switch (m_node->child1().useKind()) {
 838         case BooleanUse: {
 839             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
 840             return;
 841         }
 842
 843         case UntypedUse: {
 844             LValue value = lowJSValue(m_node->child1());
 845
 846             LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case"));
 847             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation"));
 848
 849             ValueFromBlock notBooleanResult = m_out.anchor(value);
 850             m_out.branch(isBoolean(value), unsure(booleanCase), unsure(continuation));
 851
 852             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
 853             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
 854                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
 855             m_out.jump(continuation);
 856
 857             m_out.appendTo(continuation, lastNext);
 858             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
 859             return;
 860         }
 861
 862         default:
 863             RELEASE_ASSERT_NOT_REACHED();
 864             return;
 865         }
 866     }
 867
 868     void compileGetArgument()
 869     {
 870         VariableAccessData* variable = m_node->variableAccessData();
 871         VirtualRegister operand = variable->machineLocal();
 872         RELEASE_ASSERT(operand.isArgument());
 873
 874         LValue jsValue = m_out.load64(addressFor(operand));
 875
 876         switch (useKindFor(variable->flushFormat())) {
 877         case Int32Use:
 878             speculate(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
 879             setInt32(unboxInt32(jsValue));
 880             break;
 881         case CellUse:
 882             speculate(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
 883             setJSValue(jsValue);
 884             break;
 885         case BooleanUse:
 886             speculate(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
 887             setBoolean(unboxBoolean(jsValue));
 888             break;
 889         case UntypedUse:
 890             setJSValue(jsValue);
 891             break;
 892         default:
 893             RELEASE_ASSERT_NOT_REACHED();
 894             break;
 895         }
 896     }
 897
 898     void compileExtractOSREntryLocal()
 899     {
 900         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
 901             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
 902         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
 903     }
 904
 905     void compileGetLocal()
 906     {
 907         // GetLocals arise only for captured variables.
 908
 909         VariableAccessData* variable = m_node->variableAccessData();
 910         AbstractValue& value = m_state.variables().operand(variable->local());
 911
 912         RELEASE_ASSERT(variable->isCaptured());
 913
 914         if (isInt32Speculation(value.m_type))
 915             setInt32(m_out.load32(payloadFor(variable->machineLocal())));
 916         else
 917             setJSValue(m_out.load64(addressFor(variable->machineLocal())));
 918     }
 919
 920     void compileSetLocal()
 921     {
 922         VariableAccessData* variable = m_node->variableAccessData();
 923         switch (variable->flushFormat()) {
 924         case FlushedJSValue:
 925         case FlushedArguments: {
 926             LValue value = lowJSValue(m_node->child1());
 927             m_out.store64(value, addressFor(variable->machineLocal()));
 928             break;
 929         }
 930
 931         case FlushedDouble: {
 932             LValue value = lowDouble(m_node->child1());
 933             m_out.storeDouble(value, addressFor(variable->machineLocal()));
 934             break;
 935         }
 936
 937         case FlushedInt32: {
 938             LValue value = lowInt32(m_node->child1());
 939             m_out.store32(value, payloadFor(variable->machineLocal()));
 940             break;
 941         }
 942
 943         case FlushedInt52: {
 944             LValue value = lowInt52(m_node->child1());
 945             m_out.store64(value, addressFor(variable->machineLocal()));
 946             break;
 947         }
 948
 949         case FlushedCell: {
 950             LValue value = lowCell(m_node->child1());
 951             m_out.store64(value, addressFor(variable->machineLocal()));
 952             break;
 953         }
 954
 955         case FlushedBoolean: {
 956             speculateBoolean(m_node->child1());
 957             m_out.store64(
 958                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
 959                 addressFor(variable->machineLocal()));
 960             break;
 961         }
 962
 963         default:
 964             RELEASE_ASSERT_NOT_REACHED();
 965             break;
 966         }
 967
 968         m_availability.operand(variable->local()) = Availability(variable->flushedAt());
 969     }
 970
 971     void compileMovHint()
 972     {
 973         ASSERT(m_node->containsMovHint());
 974         ASSERT(m_node->op() != ZombieHint);
 975
 976         VirtualRegister operand = m_node->unlinkedLocal();
 977         m_availability.operand(operand) = Availability(m_node->child1().node());
 978     }
 979
 980     void compileZombieHint()
 981     {
 982         m_availability.operand(m_node->unlinkedLocal()) = Availability::unavailable();
 983     }
 984
 985     void compilePhantom()
 986     {
 987         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
 988     }
 989
 990     void compileToThis()
 991     {
 992         LValue value = lowJSValue(m_node->child1());
 993
 994         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case"));
 995         LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case"));
 996         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation"));
 997
 998         m_out.branch(isCell(value), usually(isCellCase), rarely(slowCase));
 999
1000         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1001         ValueFromBlock fastResult = m_out.anchor(value);
1002         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1003
1004         m_out.appendTo(slowCase, continuation);
1005         J_JITOperation_EJ function;
1006         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1007             function = operationToThisStrict;
1008         else
1009             function = operationToThis;
1010         ValueFromBlock slowResult = m_out.anchor(
1011             vmCall(m_out.operation(function), m_callFrame, value));
1012         m_out.jump(continuation);
1013
1014         m_out.appendTo(continuation, lastNext);
1015         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1016     }
1017
1018     void compileValueAdd()
1019     {
1020         J_JITOperation_EJJ operation;
1021         if (!(m_state.forNode(m_node->child1()).m_type & SpecFullNumber)
1022             && !(m_state.forNode(m_node->child2()).m_type & SpecFullNumber))
1023             operation = operationValueAddNotNumber;
1024         else
1025             operation = operationValueAdd;
1026         setJSValue(vmCall(
1027             m_out.operation(operation), m_callFrame,
1028             lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
1029     }
1030
1031     void compileArithAddOrSub()
1032     {
1033         bool isSub =  m_node->op() == ArithSub;
1034         switch (m_node->binaryUseKind()) {
1035         case Int32Use: {
1036             LValue left = lowInt32(m_node->child1());
1037             LValue right = lowInt32(m_node->child2());
1038
1039             if (!shouldCheckOverflow(m_node->arithMode())) {
1040                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1041                 break;
1042             }
1043
1044             LValue result;
1045             if (!isSub) {
1046                 result = m_out.addWithOverflow32(left, right);
1047
1048                 if (doesKill(m_node->child2())) {
1049                     addAvailableRecovery(
1050                         m_node->child2(), SubRecovery,
1051                         m_out.extractValue(result, 0), left, ValueFormatInt32);
1052                 } else if (doesKill(m_node->child1())) {
1053                     addAvailableRecovery(
1054                         m_node->child1(), SubRecovery,
1055                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1056                 }
1057             } else {
1058                 result = m_out.subWithOverflow32(left, right);
1059
1060                 if (doesKill(m_node->child2())) {
1061                     // result = left - right
1062                     // result - left = -right
1063                     // right = left - result
1064                     addAvailableRecovery(
1065                         m_node->child2(), SubRecovery,
1066                         left, m_out.extractValue(result, 0), ValueFormatInt32);
1067                 } else if (doesKill(m_node->child1())) {
1068                     // result = left - right
1069                     // result + right = left
1070                     addAvailableRecovery(
1071                         m_node->child1(), AddRecovery,
1072                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1073                 }
1074             }
1075
1076             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
1077             setInt32(m_out.extractValue(result, 0));
1078             break;
1079         }
1080
1081         case Int52RepUse: {
1082             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)
1083                 && !m_state.forNode(m_node->child2()).couldBeType(SpecInt52)) {
1084                 Int52Kind kind;
1085                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1086                 LValue right = lowInt52(m_node->child2(), kind);
1087                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1088                 break;
1089             }
1090
1091             LValue left = lowInt52(m_node->child1());
1092             LValue right = lowInt52(m_node->child2());
1093
1094             LValue result;
1095             if (!isSub) {
1096                 result = m_out.addWithOverflow64(left, right);
1097
1098                 if (doesKill(m_node->child2())) {
1099                     addAvailableRecovery(
1100                         m_node->child2(), SubRecovery,
1101                         m_out.extractValue(result, 0), left, ValueFormatInt52);
1102                 } else if (doesKill(m_node->child1())) {
1103                     addAvailableRecovery(
1104                         m_node->child1(), SubRecovery,
1105                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1106                 }
1107             } else {
1108                 result = m_out.subWithOverflow64(left, right);
1109
1110                 if (doesKill(m_node->child2())) {
1111                     // result = left - right
1112                     // result - left = -right
1113                     // right = left - result
1114                     addAvailableRecovery(
1115                         m_node->child2(), SubRecovery,
1116                         left, m_out.extractValue(result, 0), ValueFormatInt52);
1117                 } else if (doesKill(m_node->child1())) {
1118                     // result = left - right
1119                     // result + right = left
1120                     addAvailableRecovery(
1121                         m_node->child1(), AddRecovery,
1122                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1123                 }
1124             }
1125
1126             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
1127             setInt52(m_out.extractValue(result, 0));
1128             break;
1129         }
1130
1131         case DoubleRepUse: {
1132             LValue C1 = lowDouble(m_node->child1());
1133             LValue C2 = lowDouble(m_node->child2());
1134
1135             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1136             break;
1137         }
1138
1139         default:
1140             RELEASE_ASSERT_NOT_REACHED();
1141             break;
1142         }
1143     }
1144
1145     void compileArithMul()
1146     {
1147         switch (m_node->binaryUseKind()) {
1148         case Int32Use: {
1149             LValue left = lowInt32(m_node->child1());
1150             LValue right = lowInt32(m_node->child2());
1151
1152             LValue result;
1153
1154             if (!shouldCheckOverflow(m_node->arithMode()))
1155                 result = m_out.mul(left, right);
1156             else {
1157                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
1158                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1159                 result = m_out.extractValue(overflowResult, 0);
1160             }
1161
1162             if (shouldCheckNegativeZero(m_node->arithMode())) {
1163                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1164                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1165
1166                 m_out.branch(
1167                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1168
1169                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1170                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero));
1171                 speculate(NegativeZero, noValue(), 0, cond);
1172                 m_out.jump(continuation);
1173                 m_out.appendTo(continuation, lastNext);
1174             }
1175
1176             setInt32(result);
1177             break;
1178         }
1179
1180         case Int52RepUse: {
1181             Int52Kind kind;
1182             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1183             LValue right = lowInt52(m_node->child2(), opposite(kind));
1184
1185             LValue overflowResult = m_out.mulWithOverflow64(left, right);
1186             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1187             LValue result = m_out.extractValue(overflowResult, 0);
1188
1189             if (shouldCheckNegativeZero(m_node->arithMode())) {
1190                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1191                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1192
1193                 m_out.branch(
1194                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1195
1196                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1197                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero));
1198                 speculate(NegativeZero, noValue(), 0, cond);
1199                 m_out.jump(continuation);
1200                 m_out.appendTo(continuation, lastNext);
1201             }
1202
1203             setInt52(result);
1204             break;
1205         }
1206
1207         case DoubleRepUse: {
1208             setDouble(
1209                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1210             break;
1211         }
1212
1213         default:
1214             RELEASE_ASSERT_NOT_REACHED();
1215             break;
1216         }
1217     }
1218
1219     void compileArithDiv()
1220     {
1221         switch (m_node->binaryUseKind()) {
1222         case Int32Use: {
1223             LValue numerator = lowInt32(m_node->child1());
1224             LValue denominator = lowInt32(m_node->child2());
1225
1226             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
1227             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
1228             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
1229
1230             Vector<ValueFromBlock, 3> results;
1231
1232             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1233
1234             m_out.branch(
1235                 m_out.above(adjustedDenominator, m_out.int32One),
1236                 usually(continuation), rarely(unsafeDenominator));
1237
1238             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1239
1240             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1241
1242             if (shouldCheckOverflow(m_node->arithMode())) {
1243                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1244                 speculate(Overflow, noValue(), 0, cond);
1245                 m_out.jump(continuation);
1246             } else {
1247                 // This is the case where we convert the result to an int after we're done. So,
1248                 // if the denominator is zero, then the result should be zero.
1249                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1250                 // check above) and the numerator is -2^31 then the result should be -2^31.
1251
1252                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
1253                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
1254                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
1255
1256                 m_out.branch(
1257                     m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero));
1258
1259                 m_out.appendTo(divByZero, notDivByZero);
1260                 results.append(m_out.anchor(m_out.int32Zero));
1261                 m_out.jump(done);
1262
1263                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
1264                 m_out.branch(
1265                     m_out.equal(numerator, neg2ToThe31),
1266                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1267
1268                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1269                 results.append(m_out.anchor(neg2ToThe31));
1270                 m_out.jump(done);
1271             }
1272
1273             m_out.appendTo(continuation, done);
1274
1275             if (shouldCheckNegativeZero(m_node->arithMode())) {
1276                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
1277                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
1278
1279                 m_out.branch(
1280                     m_out.isZero32(numerator),
1281                     rarely(zeroNumerator), usually(numeratorContinuation));
1282
1283                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1284
1285                 speculate(
1286                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1287
1288                 m_out.jump(numeratorContinuation);
1289
1290                 m_out.appendTo(numeratorContinuation, innerLastNext);
1291             }
1292
1293             LValue result = m_out.div(numerator, denominator);
1294
1295             if (shouldCheckOverflow(m_node->arithMode())) {
1296                 speculate(
1297                     Overflow, noValue(), 0,
1298                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1299             }
1300
1301             results.append(m_out.anchor(result));
1302             m_out.jump(done);
1303
1304             m_out.appendTo(done, lastNext);
1305
1306             setInt32(m_out.phi(m_out.int32, results));
1307             break;
1308         }
1309
1310         case DoubleRepUse: {
1311             setDouble(m_out.doubleDiv(
1312                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1313             break;
1314         }
1315
1316         default:
1317             RELEASE_ASSERT_NOT_REACHED();
1318             break;
1319         }
1320     }
1321
1322     void compileArithMod()
1323     {
1324         switch (m_node->binaryUseKind()) {
1325         case Int32Use: {
1326             LValue numerator = lowInt32(m_node->child1());
1327             LValue denominator = lowInt32(m_node->child2());
1328
1329             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
1330             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
1331             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
1332
1333             Vector<ValueFromBlock, 3> results;
1334
1335             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1336
1337             m_out.branch(
1338                 m_out.above(adjustedDenominator, m_out.int32One),
1339                 usually(continuation), rarely(unsafeDenominator));
1340
1341             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1342
1343             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1344
1345             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
1346             // separate case for that. But it probably doesn't matter so much.
1347             if (shouldCheckOverflow(m_node->arithMode())) {
1348                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1349                 speculate(Overflow, noValue(), 0, cond);
1350                 m_out.jump(continuation);
1351             } else {
1352                 // This is the case where we convert the result to an int after we're done. So,
1353                 // if the denominator is zero, then the result should be result should be zero.
1354                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1355                 // check above) and the numerator is -2^31 then the result should be -2^31.
1356
1357                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
1358                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
1359                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
1360
1361                 m_out.branch(
1362                     m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero));
1363
1364                 m_out.appendTo(modByZero, notModByZero);
1365                 results.append(m_out.anchor(m_out.int32Zero));
1366                 m_out.jump(done);
1367
1368                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
1369                 m_out.branch(
1370                     m_out.equal(numerator, neg2ToThe31),
1371                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1372
1373                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1374                 results.append(m_out.anchor(m_out.int32Zero));
1375                 m_out.jump(done);
1376             }
1377
1378             m_out.appendTo(continuation, done);
1379
1380             LValue remainder = m_out.rem(numerator, denominator);
1381
1382             if (shouldCheckNegativeZero(m_node->arithMode())) {
1383                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
1384                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
1385
1386                 m_out.branch(
1387                     m_out.lessThan(numerator, m_out.int32Zero),
1388                     unsure(negativeNumerator), unsure(numeratorContinuation));
1389
1390                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1391
1392                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1393
1394                 m_out.jump(numeratorContinuation);
1395
1396                 m_out.appendTo(numeratorContinuation, innerLastNext);
1397             }
1398
1399             results.append(m_out.anchor(remainder));
1400             m_out.jump(done);
1401
1402             m_out.appendTo(done, lastNext);
1403
1404             setInt32(m_out.phi(m_out.int32, results));
1405             break;
1406         }
1407
1408         case DoubleRepUse: {
1409             setDouble(
1410                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1411             break;
1412         }
1413
1414         default:
1415             RELEASE_ASSERT_NOT_REACHED();
1416             break;
1417         }
1418     }
1419
1420     void compileArithMinOrMax()
1421     {
1422         switch (m_node->binaryUseKind()) {
1423         case Int32Use: {
1424             LValue left = lowInt32(m_node->child1());
1425             LValue right = lowInt32(m_node->child2());
1426
1427             setInt32(
1428                 m_out.select(
1429                     m_node->op() == ArithMin
1430                         ? m_out.lessThan(left, right)
1431                         : m_out.lessThan(right, left),
1432                     left, right));
1433             break;
1434         }
1435
1436         case DoubleRepUse: {
1437             LValue left = lowDouble(m_node->child1());
1438             LValue right = lowDouble(m_node->child2());
1439
1440             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1441             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1442
1443             Vector<ValueFromBlock, 2> results;
1444
1445             results.append(m_out.anchor(left));
1446             m_out.branch(
1447                 m_node->op() == ArithMin
1448                     ? m_out.doubleLessThan(left, right)
1449                     : m_out.doubleGreaterThan(left, right),
1450                 unsure(continuation), unsure(notLessThan));
1451
1452             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1453             results.append(m_out.anchor(m_out.select(
1454                 m_node->op() == ArithMin
1455                     ? m_out.doubleGreaterThanOrEqual(left, right)
1456                     : m_out.doubleLessThanOrEqual(left, right),
1457                 right, m_out.constDouble(PNaN))));
1458             m_out.jump(continuation);
1459
1460             m_out.appendTo(continuation, lastNext);
1461             setDouble(m_out.phi(m_out.doubleType, results));
1462             break;
1463         }
1464
1465         default:
1466             RELEASE_ASSERT_NOT_REACHED();
1467             break;
1468         }
1469     }
1470
1471     void compileArithAbs()
1472     {
1473         switch (m_node->child1().useKind()) {
1474         case Int32Use: {
1475             LValue value = lowInt32(m_node->child1());
1476
1477             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1478             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1479
1480             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1481
1482             setInt32(result);
1483             break;
1484         }
1485
1486         case DoubleRepUse: {
1487             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1488             break;
1489         }
1490
1491         default:
1492             RELEASE_ASSERT_NOT_REACHED();
1493             break;
1494         }
1495     }
1496
1497     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1498
1499     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1500
1501     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1502
1503     void compileArithFRound()
1504     {
1505         LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType);
1506         setDouble(m_out.fpCast(floatValue, m_out.doubleType));
1507     }
1508
1509     void compileArithNegate()
1510     {
1511         switch (m_node->child1().useKind()) {
1512         case Int32Use: {
1513             LValue value = lowInt32(m_node->child1());
1514
1515             LValue result;
1516             if (!shouldCheckOverflow(m_node->arithMode()))
1517                 result = m_out.neg(value);
1518             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
1519                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1520                 // does the trick, though.
1521                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1522                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1523                 result = m_out.extractValue(overflowResult, 0);
1524             } else {
1525                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1526                 result = m_out.neg(value);
1527             }
1528
1529             setInt32(result);
1530             break;
1531         }
1532
1533         case Int52RepUse: {
1534             if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)) {
1535                 Int52Kind kind;
1536                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1537                 LValue result = m_out.neg(value);
1538                 if (shouldCheckNegativeZero(m_node->arithMode()))
1539                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1540                 setInt52(result, kind);
1541                 break;
1542             }
1543
1544             LValue value = lowInt52(m_node->child1());
1545             LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1546             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1547             LValue result = m_out.extractValue(overflowResult, 0);
1548             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1549             setInt52(result);
1550             break;
1551         }
1552
1553         case DoubleRepUse: {
1554             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1555             break;
1556         }
1557
1558         default:
1559             RELEASE_ASSERT_NOT_REACHED();
1560             break;
1561         }
1562     }
1563
1564     void compileBitAnd()
1565     {
1566         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1567     }
1568
1569     void compileBitOr()
1570     {
1571         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1572     }
1573
1574     void compileBitXor()
1575     {
1576         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1577     }
1578
1579     void compileBitRShift()
1580     {
1581         setInt32(m_out.aShr(
1582             lowInt32(m_node->child1()),
1583             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1584     }
1585
1586     void compileBitLShift()
1587     {
1588         setInt32(m_out.shl(
1589             lowInt32(m_node->child1()),
1590             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1591     }
1592
1593     void compileBitURShift()
1594     {
1595         setInt32(m_out.lShr(
1596             lowInt32(m_node->child1()),
1597             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1598     }
1599
1600     void compileUInt32ToNumber()
1601     {
1602         LValue value = lowInt32(m_node->child1());
1603
1604         if (doesOverflow(m_node->arithMode())) {
1605             setDouble(m_out.unsignedToDouble(value));
1606             return;
1607         }
1608
1609         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
1610         setInt32(value);
1611     }
1612
1613     void compileCheckStructure()
1614     {
1615         LValue cell = lowCell(m_node->child1());
1616
1617         ExitKind exitKind;
1618         if (m_node->child1()->op() == WeakJSConstant)
1619             exitKind = BadWeakConstantCache;
1620         else
1621             exitKind = BadCache;
1622
1623         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1624
1625         if (m_node->structureSet().size() == 1) {
1626             speculate(
1627                 exitKind, jsValueValue(cell), 0,
1628                 m_out.notEqual(structureID, weakStructure(m_node->structureSet()[0])));
1629             return;
1630         }
1631
1632         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
1633
1634         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
1635         for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
1636             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
1637             m_out.branch(
1638                 m_out.equal(structureID, weakStructure(m_node->structureSet()[i])),
1639                 unsure(continuation), unsure(nextStructure));
1640             m_out.appendTo(nextStructure);
1641         }
1642
1643         speculate(
1644             exitKind, jsValueValue(cell), 0,
1645             m_out.notEqual(structureID, weakStructure(m_node->structureSet().last())));
1646
1647         m_out.jump(continuation);
1648         m_out.appendTo(continuation, lastNext);
1649     }
1650
1651     void compileStructureTransitionWatchpoint()
1652     {
1653         addWeakReference(m_node->structure());
1654         speculateCell(m_node->child1());
1655     }
1656
1657     void compileCheckFunction()
1658     {
1659         LValue cell = lowCell(m_node->child1());
1660
1661         speculate(
1662             BadFunction, jsValueValue(cell), m_node->child1().node(),
1663             m_out.notEqual(cell, weakPointer(m_node->function())));
1664     }
1665
1666     void compileCheckExecutable()
1667     {
1668         LValue cell = lowCell(m_node->child1());
1669
1670         speculate(
1671             BadExecutable, jsValueValue(cell), m_node->child1().node(),
1672             m_out.notEqual(
1673                 m_out.loadPtr(cell, m_heaps.JSFunction_executable),
1674                 weakPointer(m_node->executable())));
1675     }
1676
1677     void compileArrayifyToStructure()
1678     {
1679         LValue cell = lowCell(m_node->child1());
1680         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
1681
1682         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
1683         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
1684
1685         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1686
1687         m_out.branch(
1688             m_out.notEqual(structureID, weakStructure(m_node->structure())),
1689             rarely(unexpectedStructure), usually(continuation));
1690
1691         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
1692
1693         if (property) {
1694             switch (m_node->arrayMode().type()) {
1695             case Array::Int32:
1696             case Array::Double:
1697             case Array::Contiguous:
1698                 speculate(
1699                     Uncountable, noValue(), 0,
1700                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
1701                 break;
1702             default:
1703                 break;
1704             }
1705         }
1706
1707         switch (m_node->arrayMode().type()) {
1708         case Array::Int32:
1709             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
1710             break;
1711         case Array::Double:
1712             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
1713             break;
1714         case Array::Contiguous:
1715             if (m_node->arrayMode().conversion() == Array::RageConvert)
1716                 vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
1717             else
1718                 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
1719             break;
1720         case Array::ArrayStorage:
1721         case Array::SlowPutArrayStorage:
1722             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
1723             break;
1724         default:
1725             RELEASE_ASSERT_NOT_REACHED();
1726             break;
1727         }
1728
1729         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
1730         speculate(
1731             BadIndexingType, jsValueValue(cell), 0,
1732             m_out.notEqual(structureID, weakStructure(m_node->structure())));
1733         m_out.jump(continuation);
1734
1735         m_out.appendTo(continuation, lastNext);
1736     }
1737
1738     void compilePutStructure()
1739     {
1740         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1741
1742         Structure* oldStructure = m_node->structureTransitionData().previousStructure;
1743         Structure* newStructure = m_node->structureTransitionData().newStructure;
1744         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
1745         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
1746         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
1747
1748         LValue cell = lowCell(m_node->child1());
1749         m_out.store32(
1750             weakStructure(newStructure),
1751             cell, m_heaps.JSCell_structureID);
1752     }
1753
1754     void compilePhantomPutStructure()
1755     {
1756         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1757     }
1758
1759     void compileGetById()
1760     {
1761         // Pretty much the only reason why we don't also support GetByIdFlush is because:
1762         // https://bugs.webkit.org/show_bug.cgi?id=125711
1763
1764         switch (m_node->child1().useKind()) {
1765         case CellUse: {
1766             setJSValue(getById(lowCell(m_node->child1())));
1767             return;
1768         }
1769
1770         case UntypedUse: {
1771             // This is pretty weird, since we duplicate the slow path both here and in the
1772             // code generated by the IC. We should investigate making this less bad.
1773             // https://bugs.webkit.org/show_bug.cgi?id=127830
1774             LValue value = lowJSValue(m_node->child1());
1775
1776             LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case"));
1777             LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case"));
1778             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation"));
1779
1780             m_out.branch(isCell(value), unsure(cellCase), unsure(notCellCase));
1781
1782             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
1783             ValueFromBlock cellResult = m_out.anchor(getById(value));
1784             m_out.jump(continuation);
1785
1786             m_out.appendTo(notCellCase, continuation);
1787             ValueFromBlock notCellResult = m_out.anchor(vmCall(
1788                 m_out.operation(operationGetById),
1789                 m_callFrame, getUndef(m_out.intPtr), value,
1790                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
1791             m_out.jump(continuation);
1792
1793             m_out.appendTo(continuation, lastNext);
1794             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
1795             return;
1796         }
1797
1798         default:
1799             RELEASE_ASSERT_NOT_REACHED();
1800             return;
1801         }
1802     }
1803
1804     void compilePutById()
1805     {
1806         // See above; CellUse is easier so we do only that for now.
1807         ASSERT(m_node->child1().useKind() == CellUse);
1808
1809         LValue base = lowCell(m_node->child1());
1810         LValue value = lowJSValue(m_node->child2());
1811         StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
1812
1813         // Arguments: id, bytes, target, numArgs, args...
1814         unsigned stackmapID = m_stackmapIDs++;
1815
1816         if (Options::verboseCompilation())
1817             dataLog("    Emitting PutById patchpoint with stackmap #", stackmapID, "\n");
1818
1819         LValue call = m_out.call(
1820             m_out.patchpointVoidIntrinsic(),
1821             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()),
1822             constNull(m_out.ref8), m_out.constInt32(2), base, value);
1823         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
1824
1825         m_ftlState.putByIds.append(PutByIdDescriptor(
1826             stackmapID, m_node->origin.semantic, uid,
1827             m_graph.executableFor(m_node->origin.semantic)->ecmaMode(),
1828             m_node->op() == PutByIdDirect ? Direct : NotDirect));
1829     }
1830
1831     void compileGetButterfly()
1832     {
1833         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1834     }
1835
1836     void compileConstantStoragePointer()
1837     {
1838         setStorage(m_out.constIntPtr(m_node->storagePointer()));
1839     }
1840
1841     void compileGetIndexedPropertyStorage()
1842     {
1843         LValue cell = lowCell(m_node->child1());
1844
1845         if (m_node->arrayMode().type() == Array::String) {
1846             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String slow case"));
1847             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String continuation"));
1848
1849             ValueFromBlock fastResult = m_out.anchor(
1850                 m_out.loadPtr(cell, m_heaps.JSString_value));
1851
1852             m_out.branch(
1853                 m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath));
1854
1855             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
1856
1857             ValueFromBlock slowResult = m_out.anchor(
1858                 vmCall(m_out.operation(operationResolveRope), m_callFrame, cell));
1859
1860             m_out.jump(continuation);
1861
1862             m_out.appendTo(continuation, lastNext);
1863
1864             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
1865             return;
1866         }
1867
1868         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
1869     }
1870
1871     void compileCheckArray()
1872     {
1873         Edge edge = m_node->child1();
1874         LValue cell = lowCell(edge);
1875
1876         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, m_state.forNode(edge)))
1877             return;
1878
1879         speculate(
1880             BadIndexingType, jsValueValue(cell), 0,
1881             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
1882     }
1883
1884     void compileGetTypedArrayByteOffset()
1885     {
1886         LValue basePtr = lowCell(m_node->child1());
1887
1888         LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array"));
1889         LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array"));
1890         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch"));
1891
1892         LValue baseAddress = m_out.addPtr(basePtr, JSArrayBufferView::offsetOfMode());
1893         m_out.branch(
1894             m_out.notEqual(baseAddress , m_out.constIntPtr(WastefulTypedArray)),
1895             unsure(simpleCase), unsure(wastefulCase));
1896
1897         // begin simple case
1898         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
1899
1900         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
1901
1902         m_out.jump(continuation);
1903
1904         // begin wasteful case
1905         m_out.appendTo(wastefulCase, continuation);
1906
1907         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
1908         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
1909         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
1910         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
1911
1912         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(dataPtr, vectorPtr));
1913
1914         m_out.jump(continuation);
1915         m_out.appendTo(continuation, lastNext);
1916
1917         // output
1918         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
1919     }
1920
1921     void compileGetMyArgumentsLength()
1922     {
1923         checkArgumentsNotCreated();
1924
1925         RELEASE_ASSERT(!m_node->origin.semantic.inlineCallFrame);
1926         setInt32(m_out.add(m_out.load32NonNegative(payloadFor(JSStack::ArgumentCount)), m_out.constInt32(-1)));
1927     }
1928
1929     void compileGetMyArgumentByVal()
1930     {
1931         checkArgumentsNotCreated();
1932
1933         CodeOrigin codeOrigin = m_node->origin.semantic;
1934
1935         LValue zeroBasedIndex = lowInt32(m_node->child1());
1936         LValue oneBasedIndex = m_out.add(zeroBasedIndex, m_out.int32One);
1937
1938         LValue limit;
1939         if (codeOrigin.inlineCallFrame)
1940             limit = m_out.constInt32(codeOrigin.inlineCallFrame->arguments.size());
1941         else
1942             limit = m_out.load32(payloadFor(JSStack::ArgumentCount));
1943
1944         speculate(Uncountable, noValue(), 0, m_out.aboveOrEqual(oneBasedIndex, limit));
1945
1946         SymbolTable* symbolTable = m_graph.baselineCodeBlockFor(codeOrigin)->symbolTable();
1947         if (symbolTable->slowArguments()) {
1948             // FIXME: FTL should support activations.
1949             // https://bugs.webkit.org/show_bug.cgi?id=129576
1950
1951             RELEASE_ASSERT_NOT_REACHED();
1952         }
1953
1954         TypedPointer base;
1955         if (codeOrigin.inlineCallFrame)
1956             base = addressFor(codeOrigin.inlineCallFrame->arguments[1].virtualRegister());
1957         else
1958             base = addressFor(virtualRegisterForArgument(1));
1959
1960         LValue pointer = m_out.baseIndex(
1961             base.value(), m_out.zeroExt(zeroBasedIndex, m_out.intPtr), ScaleEight);
1962         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
1963     }
1964
1965     void compileGetArrayLength()
1966     {
1967         switch (m_node->arrayMode().type()) {
1968         case Array::Int32:
1969         case Array::Double:
1970         case Array::Contiguous: {
1971             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1972             return;
1973         }
1974
1975         case Array::String: {
1976             LValue string = lowCell(m_node->child1());
1977             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
1978             return;
1979         }
1980
1981         default:
1982             if (isTypedView(m_node->arrayMode().typedArrayType())) {
1983                 setInt32(
1984                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
1985                 return;
1986             }
1987
1988             RELEASE_ASSERT_NOT_REACHED();
1989             return;
1990         }
1991     }
1992
1993     void compileCheckInBounds()
1994     {
1995         speculate(
1996             OutOfBounds, noValue(), 0,
1997             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1998     }
1999
2000     void compileGetByVal()
2001     {
2002         switch (m_node->arrayMode().type()) {
2003         case Array::Int32:
2004         case Array::Contiguous: {
2005             LValue index = lowInt32(m_node->child2());
2006             LValue storage = lowStorage(m_node->child3());
2007
2008             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2009                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2010
2011             if (m_node->arrayMode().isInBounds()) {
2012                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2013                 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
2014                 setJSValue(result);
2015                 return;
2016             }
2017
2018             LValue base = lowCell(m_node->child1());
2019
2020             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous fast case"));
2021             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous slow case"));
2022             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous continuation"));
2023
2024             m_out.branch(
2025                 m_out.aboveOrEqual(
2026                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2027                 rarely(slowCase), usually(fastCase));
2028
2029             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2030
2031             ValueFromBlock fastResult = m_out.anchor(
2032                 m_out.load64(baseIndex(heap, storage, index, m_node->child2())));
2033             m_out.branch(
2034                 m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation));
2035
2036             m_out.appendTo(slowCase, continuation);
2037             ValueFromBlock slowResult = m_out.anchor(
2038                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2039             m_out.jump(continuation);
2040
2041             m_out.appendTo(continuation, lastNext);
2042             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2043             return;
2044         }
2045
2046         case Array::Double: {
2047             LValue index = lowInt32(m_node->child2());
2048             LValue storage = lowStorage(m_node->child3());
2049
2050             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2051
2052             if (m_node->arrayMode().isInBounds()) {
2053                 LValue result = m_out.loadDouble(
2054                     baseIndex(heap, storage, index, m_node->child2()));
2055
2056                 if (!m_node->arrayMode().isSaneChain()) {
2057                     speculate(
2058                         LoadFromHole, noValue(), 0,
2059                         m_out.doubleNotEqualOrUnordered(result, result));
2060                 }
2061                 setDouble(result);
2062                 break;
2063             }
2064
2065             LValue base = lowCell(m_node->child1());
2066
2067             LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetByVal double in bounds"));
2068             LBasicBlock boxPath = FTL_NEW_BLOCK(m_out, ("GetByVal double boxing"));
2069             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal double slow case"));
2070             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal double continuation"));
2071
2072             m_out.branch(
2073                 m_out.aboveOrEqual(
2074                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2075                 rarely(slowCase), usually(inBounds));
2076
2077             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2078             LValue doubleValue = m_out.loadDouble(
2079                 baseIndex(heap, storage, index, m_node->child2()));
2080             m_out.branch(
2081                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2082                 rarely(slowCase), usually(boxPath));
2083
2084             m_out.appendTo(boxPath, slowCase);
2085             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2086             m_out.jump(continuation);
2087
2088             m_out.appendTo(slowCase, continuation);
2089             ValueFromBlock slowResult = m_out.anchor(
2090                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2091             m_out.jump(continuation);
2092
2093             m_out.appendTo(continuation, lastNext);
2094             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2095             return;
2096         }
2097
2098         case Array::Generic: {
2099             setJSValue(vmCall(
2100                 m_out.operation(operationGetByVal), m_callFrame,
2101                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2102             return;
2103         }
2104
2105         case Array::String: {
2106             compileStringCharAt();
2107             return;
2108         }
2109
2110         default: {
2111             LValue index = lowInt32(m_node->child2());
2112             LValue storage = lowStorage(m_node->child3());
2113
2114             TypedArrayType type = m_node->arrayMode().typedArrayType();
2115
2116             if (isTypedView(type)) {
2117                 TypedPointer pointer = TypedPointer(
2118                     m_heaps.typedArrayProperties,
2119                     m_out.add(
2120                         storage,
2121                         m_out.shl(
2122                             m_out.zeroExt(index, m_out.intPtr),
2123                             m_out.constIntPtr(logElementSize(type)))));
2124
2125                 if (isInt(type)) {
2126                     LValue result;
2127                     switch (elementSize(type)) {
2128                     case 1:
2129                         result = m_out.load8(pointer);
2130                         break;
2131                     case 2:
2132                         result = m_out.load16(pointer);
2133                         break;
2134                     case 4:
2135                         result = m_out.load32(pointer);
2136                         break;
2137                     default:
2138                         RELEASE_ASSERT_NOT_REACHED();
2139                     }
2140
2141                     if (elementSize(type) < 4) {
2142                         if (isSigned(type))
2143                             result = m_out.signExt(result, m_out.int32);
2144                         else
2145                             result = m_out.zeroExt(result, m_out.int32);
2146                         setInt32(result);
2147                         return;
2148                     }
2149
2150                     if (isSigned(type)) {
2151                         setInt32(result);
2152                         return;
2153                     }
2154
2155                     if (m_node->shouldSpeculateInt32()) {
2156                         speculate(
2157                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2158                         setInt32(result);
2159                         return;
2160                     }
2161
2162                     if (m_node->shouldSpeculateMachineInt()) {
2163                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2164                         return;
2165                     }
2166
2167                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
2168                     return;
2169                 }
2170
2171                 ASSERT(isFloat(type));
2172
2173                 LValue result;
2174                 switch (type) {
2175                 case TypeFloat32:
2176                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
2177                     break;
2178                 case TypeFloat64:
2179                     result = m_out.loadDouble(pointer);
2180                     break;
2181                 default:
2182                     RELEASE_ASSERT_NOT_REACHED();
2183                 }
2184
2185                 setDouble(result);
2186                 return;
2187             }
2188
2189             RELEASE_ASSERT_NOT_REACHED();
2190             return;
2191         } }
2192     }
2193
2194     void compilePutByVal()
2195     {
2196         Edge child1 = m_graph.varArgChild(m_node, 0);
2197         Edge child2 = m_graph.varArgChild(m_node, 1);
2198         Edge child3 = m_graph.varArgChild(m_node, 2);
2199         Edge child4 = m_graph.varArgChild(m_node, 3);
2200         Edge child5 = m_graph.varArgChild(m_node, 4);
2201
2202         switch (m_node->arrayMode().type()) {
2203         case Array::Generic: {
2204             V_JITOperation_EJJJ operation;
2205             if (m_node->op() == PutByValDirect) {
2206                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2207                     operation = operationPutByValDirectStrict;
2208                 else
2209                     operation = operationPutByValDirectNonStrict;
2210             } else {
2211                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2212                     operation = operationPutByValStrict;
2213                 else
2214                     operation = operationPutByValNonStrict;
2215             }
2216
2217             vmCall(
2218                 m_out.operation(operation), m_callFrame,
2219                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2220             return;
2221         }
2222
2223         default:
2224             break;
2225         }
2226
2227         LValue base = lowCell(child1);
2228         LValue index = lowInt32(child2);
2229         LValue storage = lowStorage(child4);
2230
2231         switch (m_node->arrayMode().type()) {
2232         case Array::Int32:
2233         case Array::Double:
2234         case Array::Contiguous: {
2235             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
2236             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2237
2238             switch (m_node->arrayMode().type()) {
2239             case Array::Int32:
2240             case Array::Contiguous: {
2241                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2242
2243                 if (m_node->arrayMode().type() == Array::Int32)
2244                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2245
2246                 TypedPointer elementPointer = m_out.baseIndex(
2247                     m_node->arrayMode().type() == Array::Int32 ?
2248                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2249                     storage, m_out.zeroExt(index, m_out.intPtr),
2250                     m_state.forNode(child2).m_value);
2251
2252                 if (m_node->op() == PutByValAlias) {
2253                     m_out.store64(value, elementPointer);
2254                     break;
2255                 }
2256
2257                 contiguousPutByValOutOfBounds(
2258                     codeBlock()->isStrictMode()
2259                     ? operationPutByValBeyondArrayBoundsStrict
2260                     : operationPutByValBeyondArrayBoundsNonStrict,
2261                     base, storage, index, value, continuation);
2262
2263                 m_out.store64(value, elementPointer);
2264                 break;
2265             }
2266
2267             case Array::Double: {
2268                 LValue value = lowDouble(child3);
2269
2270                 FTL_TYPE_CHECK(
2271                     doubleValue(value), child3, SpecDoubleReal,
2272                     m_out.doubleNotEqualOrUnordered(value, value));
2273
2274                 TypedPointer elementPointer = m_out.baseIndex(
2275                     m_heaps.indexedDoubleProperties,
2276                     storage, m_out.zeroExt(index, m_out.intPtr),
2277                     m_state.forNode(child2).m_value);
2278
2279                 if (m_node->op() == PutByValAlias) {
2280                     m_out.storeDouble(value, elementPointer);
2281                     break;
2282                 }
2283
2284                 contiguousPutByValOutOfBounds(
2285                     codeBlock()->isStrictMode()
2286                     ? operationPutDoubleByValBeyondArrayBoundsStrict
2287                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2288                     base, storage, index, value, continuation);
2289
2290                 m_out.storeDouble(value, elementPointer);
2291                 break;
2292             }
2293
2294             default:
2295                 RELEASE_ASSERT_NOT_REACHED();
2296             }
2297
2298             m_out.jump(continuation);
2299             m_out.appendTo(continuation, outerLastNext);
2300             return;
2301         }
2302
2303         default:
2304             TypedArrayType type = m_node->arrayMode().typedArrayType();
2305
2306             if (isTypedView(type)) {
2307                 TypedPointer pointer = TypedPointer(
2308                     m_heaps.typedArrayProperties,
2309                     m_out.add(
2310                         storage,
2311                         m_out.shl(
2312                             m_out.zeroExt(index, m_out.intPtr),
2313                             m_out.constIntPtr(logElementSize(type)))));
2314
2315                 LType refType;
2316                 LValue valueToStore;
2317
2318                 if (isInt(type)) {
2319                     LValue intValue;
2320                     switch (child3.useKind()) {
2321                     case Int52RepUse:
2322                     case Int32Use: {
2323                         if (child3.useKind() == Int32Use)
2324                             intValue = lowInt32(child3);
2325                         else
2326                             intValue = m_out.castToInt32(lowStrictInt52(child3));
2327
2328                         if (isClamped(type)) {
2329                             ASSERT(elementSize(type) == 1);
2330
2331                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
2332                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
2333
2334                             Vector<ValueFromBlock, 2> intValues;
2335                             intValues.append(m_out.anchor(m_out.int32Zero));
2336                             m_out.branch(
2337                                 m_out.lessThan(intValue, m_out.int32Zero),
2338                                 unsure(continuation), unsure(atLeastZero));
2339
2340                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
2341
2342                             intValues.append(m_out.anchor(m_out.select(
2343                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
2344                                 m_out.constInt32(255),
2345                                 intValue)));
2346                             m_out.jump(continuation);
2347
2348                             m_out.appendTo(continuation, lastNext);
2349                             intValue = m_out.phi(m_out.int32, intValues);
2350                         }
2351                         break;
2352                     }
2353
2354                     case DoubleRepUse: {
2355                         LValue doubleValue = lowDouble(child3);
2356
2357                         if (isClamped(type)) {
2358                             ASSERT(elementSize(type) == 1);
2359
2360                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
2361                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
2362                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
2363
2364                             Vector<ValueFromBlock, 3> intValues;
2365                             intValues.append(m_out.anchor(m_out.int32Zero));
2366                             m_out.branch(
2367                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
2368                                 unsure(continuation), unsure(atLeastZero));
2369
2370                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
2371                             intValues.append(m_out.anchor(m_out.constInt32(255)));
2372                             m_out.branch(
2373                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
2374                                 unsure(continuation), unsure(withinRange));
2375
2376                             m_out.appendTo(withinRange, continuation);
2377                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
2378                             m_out.jump(continuation);
2379
2380                             m_out.appendTo(continuation, lastNext);
2381                             intValue = m_out.phi(m_out.int32, intValues);
2382                         } else
2383                             intValue = doubleToInt32(doubleValue);
2384                         break;
2385                     }
2386
2387                     default:
2388                         RELEASE_ASSERT_NOT_REACHED();
2389                     }
2390
2391                     switch (elementSize(type)) {
2392                     case 1:
2393                         valueToStore = m_out.intCast(intValue, m_out.int8);
2394                         refType = m_out.ref8;
2395                         break;
2396                     case 2:
2397                         valueToStore = m_out.intCast(intValue, m_out.int16);
2398                         refType = m_out.ref16;
2399                         break;
2400                     case 4:
2401                         valueToStore = intValue;
2402                         refType = m_out.ref32;
2403                         break;
2404                     default:
2405                         RELEASE_ASSERT_NOT_REACHED();
2406                     }
2407                 } else /* !isInt(type) */ {
2408                     LValue value = lowDouble(child3);
2409                     switch (type) {
2410                     case TypeFloat32:
2411                         valueToStore = m_out.fpCast(value, m_out.floatType);
2412                         refType = m_out.refFloat;
2413                         break;
2414                     case TypeFloat64:
2415                         valueToStore = value;
2416                         refType = m_out.refDouble;
2417                         break;
2418                     default:
2419                         RELEASE_ASSERT_NOT_REACHED();
2420                     }
2421                 }
2422
2423                 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
2424                     m_out.store(valueToStore, pointer, refType);
2425                 else {
2426                     LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case"));
2427                     LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation"));
2428
2429                     m_out.branch(
2430                         m_out.aboveOrEqual(index, lowInt32(child5)),
2431                         unsure(continuation), unsure(isInBounds));
2432
2433                     LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
2434                     m_out.store(valueToStore, pointer, refType);
2435                     m_out.jump(continuation);
2436
2437                     m_out.appendTo(continuation, lastNext);
2438                 }
2439
2440                 return;
2441             }
2442
2443             RELEASE_ASSERT_NOT_REACHED();
2444             break;
2445         }
2446     }
2447
2448     void compileArrayPush()
2449     {
2450         LValue base = lowCell(m_node->child1());
2451         LValue storage = lowStorage(m_node->child3());
2452
2453         switch (m_node->arrayMode().type()) {
2454         case Array::Int32:
2455         case Array::Contiguous:
2456         case Array::Double: {
2457             LValue value;
2458             LType refType;
2459
2460             if (m_node->arrayMode().type() != Array::Double) {
2461                 value = lowJSValue(m_node->child2(), ManualOperandSpeculation);
2462                 if (m_node->arrayMode().type() == Array::Int32) {
2463                     FTL_TYPE_CHECK(
2464                         jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value));
2465                 }
2466                 refType = m_out.ref64;
2467             } else {
2468                 value = lowDouble(m_node->child2());
2469                 FTL_TYPE_CHECK(
2470                     doubleValue(value), m_node->child2(), SpecDoubleReal,
2471                     m_out.doubleNotEqualOrUnordered(value, value));
2472                 refType = m_out.refDouble;
2473             }
2474
2475             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2476
2477             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2478
2479             LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path"));
2480             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path"));
2481             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation"));
2482
2483             m_out.branch(
2484                 m_out.aboveOrEqual(
2485                     prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
2486                 rarely(slowPath), usually(fastPath));
2487
2488             LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
2489             m_out.store(
2490                 value,
2491                 m_out.baseIndex(heap, storage, m_out.zeroExt(prevLength, m_out.intPtr)),
2492                 refType);
2493             LValue newLength = m_out.add(prevLength, m_out.int32One);
2494             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2495
2496             ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
2497             m_out.jump(continuation);
2498
2499             m_out.appendTo(slowPath, continuation);
2500             LValue operation;
2501             if (m_node->arrayMode().type() != Array::Double)
2502                 operation = m_out.operation(operationArrayPush);
2503             else
2504                 operation = m_out.operation(operationArrayPushDouble);
2505             ValueFromBlock slowResult = m_out.anchor(
2506                 vmCall(operation, m_callFrame, value, base));
2507             m_out.jump(continuation);
2508
2509             m_out.appendTo(continuation, lastNext);
2510             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2511             return;
2512         }
2513
2514         default:
2515             RELEASE_ASSERT_NOT_REACHED();
2516             return;
2517         }
2518     }
2519
2520     void compileArrayPop()
2521     {
2522         LValue base = lowCell(m_node->child1());
2523         LValue storage = lowStorage(m_node->child2());
2524
2525         switch (m_node->arrayMode().type()) {
2526         case Array::Int32:
2527         case Array::Double:
2528         case Array::Contiguous: {
2529             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2530
2531             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case"));
2532             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case"));
2533             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation"));
2534
2535             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2536
2537             Vector<ValueFromBlock, 3> results;
2538             results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
2539             m_out.branch(
2540                 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
2541
2542             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2543             LValue newLength = m_out.sub(prevLength, m_out.int32One);
2544             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2545             TypedPointer pointer = m_out.baseIndex(
2546                 heap, storage, m_out.zeroExt(newLength, m_out.intPtr));
2547             if (m_node->arrayMode().type() != Array::Double) {
2548                 LValue result = m_out.load64(pointer);
2549                 m_out.store64(m_out.int64Zero, pointer);
2550                 results.append(m_out.anchor(result));
2551                 m_out.branch(
2552                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
2553             } else {
2554                 LValue result = m_out.loadDouble(pointer);
2555                 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
2556                 results.append(m_out.anchor(boxDouble(result)));
2557                 m_out.branch(
2558                     m_out.doubleEqual(result, result),
2559                     usually(continuation), rarely(slowCase));
2560             }
2561
2562             m_out.appendTo(slowCase, continuation);
2563             results.append(m_out.anchor(vmCall(
2564                 m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base)));
2565             m_out.jump(continuation);
2566
2567             m_out.appendTo(continuation, lastNext);
2568             setJSValue(m_out.phi(m_out.int64, results));
2569             return;
2570         }
2571
2572         default:
2573             RELEASE_ASSERT_NOT_REACHED();
2574             return;
2575         }
2576     }
2577
2578     void compileNewObject()
2579     {
2580         Structure* structure = m_node->structure();
2581         size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
2582         MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize);
2583
2584         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewObject slow path"));
2585         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewObject continuation"));
2586
2587         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2588
2589         ValueFromBlock fastResult = m_out.anchor(allocateObject(
2590             m_out.constIntPtr(allocator), structure, m_out.intPtrZero, slowPath));
2591
2592         m_out.jump(continuation);
2593
2594         m_out.appendTo(slowPath, continuation);
2595
2596         ValueFromBlock slowResult = m_out.anchor(vmCall(
2597             m_out.operation(operationNewObject), m_callFrame, m_out.constIntPtr(structure)));
2598         m_out.jump(continuation);
2599
2600         m_out.appendTo(continuation, lastNext);
2601         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2602     }
2603
2604     void compileNewArray()
2605     {
2606         // First speculate appropriately on all of the children. Do this unconditionally up here
2607         // because some of the slow paths may otherwise forget to do it. It's sort of arguable
2608         // that doing the speculations up here might be unprofitable for RA - so we can consider
2609         // sinking this to below the allocation fast path if we find that this has a lot of
2610         // register pressure.
2611         for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex)
2612             speculate(m_graph.varArgChild(m_node, operandIndex));
2613
2614         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2615         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
2616             m_node->indexingType());
2617
2618         RELEASE_ASSERT(structure->indexingType() == m_node->indexingType());
2619
2620         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
2621             unsigned numElements = m_node->numChildren();
2622
2623             ArrayValues arrayValues = allocateJSArray(structure, numElements);
2624
2625             for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
2626                 Edge edge = m_graph.varArgChild(m_node, operandIndex);
2627
2628                 switch (m_node->indexingType()) {
2629                 case ALL_BLANK_INDEXING_TYPES:
2630                 case ALL_UNDECIDED_INDEXING_TYPES:
2631                     CRASH();
2632                     break;
2633
2634                 case ALL_DOUBLE_INDEXING_TYPES:
2635                     m_out.storeDouble(
2636                         lowDouble(edge),
2637                         arrayValues.butterfly, m_heaps.indexedDoubleProperties[operandIndex]);
2638                     break;
2639
2640                 case ALL_INT32_INDEXING_TYPES:
2641                 case ALL_CONTIGUOUS_INDEXING_TYPES:
2642                     m_out.store64(
2643                         lowJSValue(edge, ManualOperandSpeculation),
2644                         arrayValues.butterfly,
2645                         m_heaps.forIndexingType(m_node->indexingType())->at(operandIndex));
2646                     break;
2647
2648                 default:
2649                     CRASH();
2650                 }
2651             }
2652
2653             setJSValue(arrayValues.array);
2654             return;
2655         }
2656
2657         if (!m_node->numChildren()) {
2658             setJSValue(vmCall(
2659                 m_out.operation(operationNewEmptyArray), m_callFrame,
2660                 m_out.constIntPtr(structure)));
2661             return;
2662         }
2663
2664         size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren();
2665         ASSERT(scratchSize);
2666         ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
2667         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
2668
2669         for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
2670             Edge edge = m_graph.varArgChild(m_node, operandIndex);
2671             m_out.store64(
2672                 lowJSValue(edge, ManualOperandSpeculation),
2673                 m_out.absolute(buffer + operandIndex));
2674         }
2675
2676         m_out.storePtr(
2677             m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->activeLengthPtr()));
2678
2679         LValue result = vmCall(
2680             m_out.operation(operationNewArray), m_callFrame,
2681             m_out.constIntPtr(structure), m_out.constIntPtr(buffer),
2682             m_out.constIntPtr(m_node->numChildren()));
2683
2684         m_out.storePtr(m_out.intPtrZero, m_out.absolute(scratchBuffer->activeLengthPtr()));
2685
2686         setJSValue(result);
2687     }
2688
2689     void compileNewArrayBuffer()
2690     {
2691         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2692         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
2693             m_node->indexingType());
2694
2695         RELEASE_ASSERT(structure->indexingType() == m_node->indexingType());
2696
2697         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
2698             unsigned numElements = m_node->numConstants();
2699
2700             ArrayValues arrayValues = allocateJSArray(structure, numElements);
2701
2702             JSValue* data = codeBlock()->constantBuffer(m_node->startConstant());
2703             for (unsigned index = 0; index < m_node->numConstants(); ++index) {
2704                 int64_t value;
2705                 if (hasDouble(m_node->indexingType()))
2706                     value = bitwise_cast<int64_t>(data[index].asNumber());
2707                 else
2708                     value = JSValue::encode(data[index]);
2709
2710                 m_out.store64(
2711                     m_out.constInt64(value),
2712                     arrayValues.butterfly,
2713                     m_heaps.forIndexingType(m_node->indexingType())->at(index));
2714             }
2715
2716             setJSValue(arrayValues.array);
2717             return;
2718         }
2719
2720         setJSValue(vmCall(
2721             m_out.operation(operationNewArrayBuffer), m_callFrame,
2722             m_out.constIntPtr(structure), m_out.constIntPtr(m_node->startConstant()),
2723             m_out.constIntPtr(m_node->numConstants())));
2724     }
2725
2726     void compileNewArrayWithSize()
2727     {
2728         LValue publicLength = lowInt32(m_node->child1());
2729
2730         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2731         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
2732             m_node->indexingType());
2733
2734         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
2735             ASSERT(
2736                 hasUndecided(structure->indexingType())
2737                 || hasInt32(structure->indexingType())
2738                 || hasDouble(structure->indexingType())
2739                 || hasContiguous(structure->indexingType()));
2740
2741             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fast case"));
2742             LBasicBlock largeCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize large case"));
2743             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fail case"));
2744             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize slow case"));
2745             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize continuation"));
2746
2747             m_out.branch(
2748                 m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)),
2749                 rarely(largeCase), usually(fastCase));
2750
2751             LBasicBlock lastNext = m_out.appendTo(fastCase, largeCase);
2752
2753             // We don't round up to BASE_VECTOR_LEN for new Array(blah).
2754             LValue vectorLength = publicLength;
2755
2756             LValue payloadSize =
2757                 m_out.shl(m_out.zeroExt(vectorLength, m_out.intPtr), m_out.constIntPtr(3));
2758
2759             LValue butterflySize = m_out.add(
2760                 payloadSize, m_out.constIntPtr(sizeof(IndexingHeader)));
2761
2762             LValue endOfStorage = allocateBasicStorageAndGetEnd(butterflySize, failCase);
2763
2764             LValue butterfly = m_out.sub(endOfStorage, payloadSize);
2765
2766             LValue object = allocateObject<JSArray>(
2767                 structure, butterfly, failCase);
2768
2769             m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength);
2770             m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength);
2771
2772             if (hasDouble(m_node->indexingType())) {
2773                 LBasicBlock initLoop = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init loop"));
2774                 LBasicBlock initDone = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init done"));
2775
2776                 ValueFromBlock originalIndex = m_out.anchor(vectorLength);
2777                 ValueFromBlock originalPointer = m_out.anchor(butterfly);
2778                 m_out.branch(
2779                     m_out.notZero32(vectorLength), unsure(initLoop), unsure(initDone));
2780
2781                 LBasicBlock initLastNext = m_out.appendTo(initLoop, initDone);
2782                 LValue index = m_out.phi(m_out.int32, originalIndex);
2783                 LValue pointer = m_out.phi(m_out.intPtr, originalPointer);
2784
2785                 m_out.store64(
2786                     m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
2787                     TypedPointer(m_heaps.indexedDoubleProperties.atAnyIndex(), pointer));
2788
2789                 LValue nextIndex = m_out.sub(index, m_out.int32One);
2790                 addIncoming(index, m_out.anchor(nextIndex));
2791                 addIncoming(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight)));
2792                 m_out.branch(
2793                     m_out.notZero32(nextIndex), unsure(initLoop), unsure(initDone));
2794
2795                 m_out.appendTo(initDone, initLastNext);
2796             }
2797
2798             ValueFromBlock fastResult = m_out.anchor(object);
2799             m_out.jump(continuation);
2800
2801             m_out.appendTo(largeCase, failCase);
2802             ValueFromBlock largeStructure = m_out.anchor(m_out.constIntPtr(
2803                 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)));
2804             m_out.jump(slowCase);
2805
2806             m_out.appendTo(failCase, slowCase);
2807             ValueFromBlock failStructure = m_out.anchor(m_out.constIntPtr(structure));
2808             m_out.jump(slowCase);
2809
2810             m_out.appendTo(slowCase, continuation);
2811             LValue structureValue = m_out.phi(
2812                 m_out.intPtr, largeStructure, failStructure);
2813             ValueFromBlock slowResult = m_out.anchor(vmCall(
2814                 m_out.operation(operationNewArrayWithSize),
2815                 m_callFrame, structureValue, publicLength));
2816             m_out.jump(continuation);
2817
2818             m_out.appendTo(continuation, lastNext);
2819             setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
2820             return;
2821         }
2822
2823         LValue structureValue = m_out.select(
2824             m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)),
2825             m_out.constIntPtr(
2826                 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)),
2827             m_out.constIntPtr(structure));
2828         setJSValue(vmCall(m_out.operation(operationNewArrayWithSize), m_callFrame, structureValue, publicLength));
2829     }
2830
2831     void compileAllocatePropertyStorage()
2832     {
2833         StructureTransitionData& data = m_node->structureTransitionData();
2834         LValue object = lowCell(m_node->child1());
2835
2836         setStorage(allocatePropertyStorage(object, data.previousStructure));
2837     }
2838
2839     void compileReallocatePropertyStorage()
2840     {
2841         StructureTransitionData& data = m_node->structureTransitionData();
2842         LValue object = lowCell(m_node->child1());
2843         LValue oldStorage = lowStorage(m_node->child2());
2844
2845         setStorage(
2846             reallocatePropertyStorage(
2847                 object, oldStorage, data.previousStructure, data.newStructure));
2848     }
2849
2850     void compileToString()
2851     {
2852         switch (m_node->child1().useKind()) {
2853         case StringObjectUse: {
2854             LValue cell = lowCell(m_node->child1());
2855             speculateStringObjectForCell(m_node->child1(), cell);
2856             m_interpreter.filter(m_node->child1(), SpecStringObject);
2857
2858             setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
2859             return;
2860         }
2861
2862         case StringOrStringObjectUse: {
2863             LValue cell = lowCell(m_node->child1());
2864             LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2865
2866             LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject not string case"));
2867             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject continuation"));
2868
2869             ValueFromBlock simpleResult = m_out.anchor(cell);
2870             m_out.branch(
2871                 m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
2872                 unsure(continuation), unsure(notString));
2873
2874             LBasicBlock lastNext = m_out.appendTo(notString, continuation);
2875             speculateStringObjectForStructureID(m_node->child1(), structureID);
2876             ValueFromBlock unboxedResult = m_out.anchor(
2877                 m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
2878             m_out.jump(continuation);
2879
2880             m_out.appendTo(continuation, lastNext);
2881             setJSValue(m_out.phi(m_out.int64, simpleResult, unboxedResult));
2882
2883             m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject);
2884             return;
2885         }
2886
2887         case CellUse:
2888         case UntypedUse: {
2889             LValue value;
2890             if (m_node->child1().useKind() == CellUse)
2891                 value = lowCell(m_node->child1());
2892             else
2893                 value = lowJSValue(m_node->child1());
2894
2895             LBasicBlock isCell = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse is cell"));
2896             LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse not string"));
2897             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse continuation"));
2898
2899             LValue isCellPredicate;
2900             if (m_node->child1().useKind() == CellUse)
2901                 isCellPredicate = m_out.booleanTrue;
2902             else
2903                 isCellPredicate = this->isCell(value);
2904             m_out.branch(isCellPredicate, unsure(isCell), unsure(notString));
2905
2906             LBasicBlock lastNext = m_out.appendTo(isCell, notString);
2907             ValueFromBlock simpleResult = m_out.anchor(value);
2908             LValue isStringPredicate;
2909             if (m_node->child1()->prediction() & SpecString) {
2910                 isStringPredicate = m_out.equal(
2911                     m_out.load32(value, m_heaps.JSCell_structureID),
2912                     m_out.constInt32(vm().stringStructure->id()));
2913             } else
2914                 isStringPredicate = m_out.booleanFalse;
2915             m_out.branch(isStringPredicate, unsure(continuation), unsure(notString));
2916
2917             m_out.appendTo(notString, continuation);
2918             LValue operation;
2919             if (m_node->child1().useKind() == CellUse)
2920                 operation = m_out.operation(operationToStringOnCell);
2921             else
2922                 operation = m_out.operation(operationToString);
2923             ValueFromBlock convertedResult = m_out.anchor(vmCall(operation, m_callFrame, value));
2924             m_out.jump(continuation);
2925
2926             m_out.appendTo(continuation, lastNext);
2927             setJSValue(m_out.phi(m_out.int64, simpleResult, convertedResult));
2928             return;
2929         }
2930
2931         default:
2932             RELEASE_ASSERT_NOT_REACHED();
2933             break;
2934         }
2935     }
2936
2937     void compileToPrimitive()
2938     {
2939         LValue value = lowJSValue(m_node->child1());
2940
2941         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive cell case"));
2942         LBasicBlock isObjectCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive object case"));
2943         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToPrimitive continuation"));
2944
2945         Vector<ValueFromBlock, 3> results;
2946
2947         results.append(m_out.anchor(value));
2948         m_out.branch(isCell(value), unsure(isCellCase), unsure(continuation));
2949
2950         LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase);
2951         results.append(m_out.anchor(value));
2952         m_out.branch(isObject(value), unsure(isObjectCase), unsure(continuation));
2953
2954         m_out.appendTo(isObjectCase, continuation);
2955         results.append(m_out.anchor(vmCall(
2956             m_out.operation(operationToPrimitive), m_callFrame, value)));
2957         m_out.jump(continuation);
2958
2959         m_out.appendTo(continuation, lastNext);
2960         setJSValue(m_out.phi(m_out.int64, results));
2961     }
2962
2963     void compileMakeRope()
2964     {
2965         LValue kids[3];
2966         unsigned numKids;
2967         kids[0] = lowCell(m_node->child1());
2968         kids[1] = lowCell(m_node->child2());
2969         if (m_node->child3()) {
2970             kids[2] = lowCell(m_node->child3());
2971             numKids = 3;
2972         } else {
2973             kids[2] = 0;
2974             numKids = 2;
2975         }
2976
2977         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MakeRope slow path"));
2978         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MakeRope continuation"));
2979
2980         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
2981
2982         MarkedAllocator& allocator =
2983             vm().heap.allocatorForObjectWithImmortalStructureDestructor(sizeof(JSRopeString));
2984
2985         LValue result = allocateCell(
2986             m_out.constIntPtr(&allocator),
2987             vm().stringStructure.get(),
2988             slowPath);
2989
2990         m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSString_value);
2991         for (unsigned i = 0; i < numKids; ++i)
2992             m_out.storePtr(kids[i], result, m_heaps.JSRopeString_fibers[i]);
2993         for (unsigned i = numKids; i < JSRopeString::s_maxInternalRopeLength; ++i)
2994             m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSRopeString_fibers[i]);
2995         LValue flags = m_out.load32(kids[0], m_heaps.JSString_flags);
2996         LValue length = m_out.load32(kids[0], m_heaps.JSString_length);
2997         for (unsigned i = 1; i < numKids; ++i) {
2998             flags = m_out.bitAnd(flags, m_out.load32(kids[i], m_heaps.JSString_flags));
2999             LValue lengthAndOverflow = m_out.addWithOverflow32(
3000                 length, m_out.load32(kids[i], m_heaps.JSString_length));
3001             speculate(Uncountable, noValue(), 0, m_out.extractValue(lengthAndOverflow, 1));
3002             length = m_out.extractValue(lengthAndOverflow, 0);
3003         }
3004         m_out.store32(
3005             m_out.bitAnd(m_out.constInt32(JSString::Is8Bit), flags),
3006             result, m_heaps.JSString_flags);
3007         m_out.store32(length, result, m_heaps.JSString_length);
3008
3009         ValueFromBlock fastResult = m_out.anchor(result);
3010         m_out.jump(continuation);
3011
3012         m_out.appendTo(slowPath, continuation);
3013         ValueFromBlock slowResult;
3014         switch (numKids) {
3015         case 2:
3016             slowResult = m_out.anchor(vmCall(
3017                 m_out.operation(operationMakeRope2), m_callFrame, kids[0], kids[1]));
3018             break;
3019         case 3:
3020             slowResult = m_out.anchor(vmCall(
3021                 m_out.operation(operationMakeRope3), m_callFrame, kids[0], kids[1], kids[2]));
3022             break;
3023         default:
3024             RELEASE_ASSERT_NOT_REACHED();
3025             break;
3026         }
3027         m_out.jump(continuation);
3028
3029         m_out.appendTo(continuation, lastNext);
3030         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
3031     }
3032
3033     void compileStringCharAt()
3034     {
3035         LValue base = lowCell(m_node->child1());
3036         LValue index = lowInt32(m_node->child2());
3037         LValue storage = lowStorage(m_node->child3());
3038
3039         LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("GetByVal String fast path"));
3040         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetByVal String slow path"));
3041         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal String continuation"));
3042
3043         m_out.branch(
3044             m_out.aboveOrEqual(
3045                 index, m_out.load32NonNegative(base, m_heaps.JSString_length)),
3046             rarely(slowPath), usually(fastPath));
3047
3048         LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
3049
3050         LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3051
3052         LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 8-bit case"));
3053         LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 16-bit case"));
3054         LBasicBlock bitsContinuation = FTL_NEW_BLOCK(m_out, ("GetByVal String bitness continuation"));
3055         LBasicBlock bigCharacter = FTL_NEW_BLOCK(m_out, ("GetByVal String big character"));
3056
3057         m_out.branch(
3058             m_out.testIsZero32(
3059                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3060                 m_out.constInt32(StringImpl::flagIs8Bit())),
3061             unsure(is16Bit), unsure(is8Bit));
3062
3063         m_out.appendTo(is8Bit, is16Bit);
3064
3065         ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3066             m_out.load8(m_out.baseIndex(
3067                 m_heaps.characters8,
3068                 storage, m_out.zeroExt(index, m_out.intPtr),
3069                 m_state.forNode(m_node->child2()).m_value)),
3070             m_out.int32));
3071         m_out.jump(bitsContinuation);
3072
3073         m_out.appendTo(is16Bit, bigCharacter);
3074
3075         ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3076             m_out.load16(m_out.baseIndex(
3077                 m_heaps.characters16,
3078                 storage, m_out.zeroExt(index, m_out.intPtr),
3079                 m_state.forNode(m_node->child2()).m_value)),
3080             m_out.int32));
3081         m_out.branch(
3082             m_out.aboveOrEqual(char16Bit.value(), m_out.constInt32(0x100)),
3083             rarely(bigCharacter), usually(bitsContinuation));
3084
3085         m_out.appendTo(bigCharacter, bitsContinuation);
3086
3087         Vector<ValueFromBlock, 4> results;
3088         results.append(m_out.anchor(vmCall(
3089             m_out.operation(operationSingleCharacterString),
3090             m_callFrame, char16Bit.value())));
3091         m_out.jump(continuation);
3092
3093         m_out.appendTo(bitsContinuation, slowPath);
3094
3095         LValue character = m_out.phi(m_out.int32, char8Bit, char16Bit);
3096
3097         LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
3098
3099         results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex(
3100             m_heaps.singleCharacterStrings, smallStrings,
3101             m_out.zeroExt(character, m_out.intPtr)))));
3102         m_out.jump(continuation);
3103
3104         m_out.appendTo(slowPath, continuation);
3105
3106         if (m_node->arrayMode().isInBounds()) {
3107             speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue);
3108             results.append(m_out.anchor(m_out.intPtrZero));
3109         } else {
3110             JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3111
3112             if (globalObject->stringPrototypeChainIsSane()) {
3113                 LBasicBlock negativeIndex = FTL_NEW_BLOCK(m_out, ("GetByVal String negative index"));
3114
3115                 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
3116                 m_out.branch(
3117                     m_out.lessThan(index, m_out.int32Zero),
3118                     rarely(negativeIndex), usually(continuation));
3119
3120                 m_out.appendTo(negativeIndex, continuation);
3121             }
3122
3123             results.append(m_out.anchor(vmCall(
3124                 m_out.operation(operationGetByValStringInt), m_callFrame, base, index)));
3125         }
3126
3127         m_out.jump(continuation);
3128
3129         m_out.appendTo(continuation, lastNext);
3130         setJSValue(m_out.phi(m_out.int64, results));
3131     }
3132
3133     void compileStringCharCodeAt()
3134     {
3135         LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 8-bit case"));
3136         LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 16-bit case"));
3137         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt continuation"));
3138
3139         LValue base = lowCell(m_node->child1());
3140         LValue index = lowInt32(m_node->child2());
3141         LValue storage = lowStorage(m_node->child3());
3142
3143         speculate(
3144             Uncountable, noValue(), 0,
3145             m_out.aboveOrEqual(
3146                 index, m_out.load32NonNegative(base, m_heaps.JSString_length)));
3147
3148         LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3149
3150         m_out.branch(
3151             m_out.testIsZero32(
3152                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3153                 m_out.constInt32(StringImpl::flagIs8Bit())),
3154             unsure(is16Bit), unsure(is8Bit));
3155
3156         LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
3157
3158         ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3159             m_out.load8(m_out.baseIndex(
3160                 m_heaps.characters8,
3161                 storage, m_out.zeroExt(index, m_out.intPtr),
3162                 m_state.forNode(m_node->child2()).m_value)),
3163             m_out.int32));
3164         m_out.jump(continuation);
3165
3166         m_out.appendTo(is16Bit, continuation);
3167
3168         ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3169             m_out.load16(m_out.baseIndex(
3170                 m_heaps.characters16,
3171                 storage, m_out.zeroExt(index, m_out.intPtr),
3172                 m_state.forNode(m_node->child2()).m_value)),
3173             m_out.int32));
3174         m_out.jump(continuation);
3175
3176         m_out.appendTo(continuation, lastNext);
3177
3178         setInt32(m_out.phi(m_out.int32, char8Bit, char16Bit));
3179     }
3180
3181     void compileGetByOffset()
3182     {
3183         StorageAccessData& data =
3184             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
3185
3186         setJSValue(loadProperty(
3187             lowStorage(m_node->child1()), data.identifierNumber, data.offset));
3188     }
3189
3190     void compileMultiGetByOffset()
3191     {
3192         LValue base = lowCell(m_node->child1());
3193
3194         MultiGetByOffsetData& data = m_node->multiGetByOffsetData();
3195
3196         Vector<LBasicBlock, 2> blocks(data.variants.size());
3197         for (unsigned i = data.variants.size(); i--;)
3198             blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset case ", i));
3199         LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset fail"));
3200         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset continuation"));
3201
3202         Vector<SwitchCase, 2> cases;
3203         for (unsigned i = data.variants.size(); i--;) {
3204             GetByIdVariant variant = data.variants[i];
3205             for (unsigned j = variant.structureSet().size(); j--;) {
3206                 cases.append(SwitchCase(
3207                     weakStructure(variant.structureSet()[j]), blocks[i], Weight(1)));
3208             }
3209         }
3210         m_out.switchInstruction(
3211             m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3212
3213         LBasicBlock lastNext = m_out.m_nextBlock;
3214
3215         Vector<ValueFromBlock, 2> results;
3216         for (unsigned i = data.variants.size(); i--;) {
3217             m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
3218
3219             GetByIdVariant variant = data.variants[i];
3220             LValue result;
3221             if (variant.specificValue())
3222                 result = m_out.constInt64(JSValue::encode(variant.specificValue()));
3223             else {
3224                 LValue propertyBase;
3225                 if (variant.chain())
3226                     propertyBase = weakPointer(variant.chain()->terminalPrototype());
3227                 else
3228                     propertyBase = base;
3229                 if (!isInlineOffset(variant.offset()))
3230                     propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly);
3231                 result = loadProperty(propertyBase, data.identifierNumber, variant.offset());
3232             }
3233
3234             results.append(m_out.anchor(result));
3235             m_out.jump(continuation);
3236         }
3237
3238         m_out.appendTo(exit, continuation);
3239         terminate(BadCache);
3240         m_out.unreachable();
3241
3242         m_out.appendTo(continuation, lastNext);
3243         setJSValue(m_out.phi(m_out.int64, results));
3244     }
3245
3246     void compilePutByOffset()
3247     {
3248         StorageAccessData& data =
3249             m_graph.m_storageAccessData[m_node->storageAccessDataIndex()];
3250
3251         storeProperty(
3252             lowJSValue(m_node->child3()),
3253             lowStorage(m_node->child1()), data.identifierNumber, data.offset);
3254     }
3255
3256     void compileMultiPutByOffset()
3257     {
3258         LValue base = lowCell(m_node->child1());
3259         LValue value = lowJSValue(m_node->child2());
3260
3261         MultiPutByOffsetData& data = m_node->multiPutByOffsetData();
3262
3263         Vector<LBasicBlock, 2> blocks(data.variants.size());
3264         for (unsigned i = data.variants.size(); i--;)
3265             blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset case ", i));
3266         LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset fail"));
3267         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset continuation"));
3268
3269         Vector<SwitchCase, 2> cases;
3270         for (unsigned i = data.variants.size(); i--;) {
3271             PutByIdVariant variant = data.variants[i];
3272             cases.append(
3273                 SwitchCase(weakStructure(variant.oldStructure()), blocks[i], Weight(1)));
3274         }
3275         m_out.switchInstruction(
3276             m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3277
3278         LBasicBlock lastNext = m_out.m_nextBlock;
3279
3280         for (unsigned i = data.variants.size(); i--;) {
3281             m_out.appendTo(blocks[i], i  + 1 < data.variants.size() ? blocks[i + 1] : exit);
3282
3283             PutByIdVariant variant = data.variants[i];
3284
3285             LValue storage;
3286             if (variant.kind() == PutByIdVariant::Replace) {
3287                 if (isInlineOffset(variant.offset()))
3288                     storage = base;
3289                 else
3290                     storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
3291             } else {
3292                 m_graph.m_plan.transitions.addLazily(
3293                     codeBlock(), m_node->origin.semantic.codeOriginOwner(),
3294                     variant.oldStructure(), variant.newStructure());
3295
3296                 storage = storageForTransition(
3297                     base, variant.offset(), variant.oldStructure(), variant.newStructure());
3298
3299                 ASSERT(variant.oldStructure()->indexingType() == variant.newStructure()->indexingType());
3300                 ASSERT(variant.oldStructure()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags());
3301                 ASSERT(variant.oldStructure()->typeInfo().type() == variant.newStructure()->typeInfo().type());
3302                 m_out.store32(
3303                     weakStructure(variant.newStructure()), base, m_heaps.JSCell_structureID);
3304             }
3305
3306             storeProperty(value, storage, data.identifierNumber, variant.offset());
3307             m_out.jump(continuation);
3308         }
3309
3310         m_out.appendTo(exit, continuation);
3311         terminate(BadCache);
3312         m_out.unreachable();
3313
3314         m_out.appendTo(continuation, lastNext);
3315     }
3316
3317     void compileGetGlobalVar()
3318     {
3319         setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer())));
3320     }
3321
3322     void compilePutGlobalVar()
3323     {
3324         m_out.store64(
3325             lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer()));
3326     }
3327
3328     void compileNotifyWrite()
3329     {
3330         VariableWatchpointSet* set = m_node->variableWatchpointSet();
3331
3332         LValue value = lowJSValue(m_node->child1());
3333
3334         LBasicBlock isNotInvalidated = FTL_NEW_BLOCK(m_out, ("NotifyWrite not invalidated case"));
3335         LBasicBlock notifySlow = FTL_NEW_BLOCK(m_out, ("NotifyWrite notify slow case"));
3336         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NotifyWrite continuation"));
3337
3338         LValue state = m_out.load8(m_out.absolute(set->addressOfState()));
3339
3340         m_out.branch(
3341             m_out.equal(state, m_out.constInt8(IsInvalidated)),
3342             usually(continuation), rarely(isNotInvalidated));
3343
3344         LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, notifySlow);
3345
3346         m_out.branch(
3347             m_out.equal(value, m_out.load64(m_out.absolute(set->addressOfInferredValue()))),
3348             unsure(continuation), unsure(notifySlow));
3349
3350         m_out.appendTo(notifySlow, continuation);
3351
3352         vmCall(m_out.operation(operationNotifyWrite), m_callFrame, m_out.constIntPtr(set), value);
3353         m_out.jump(continuation);
3354
3355         m_out.appendTo(continuation, lastNext);
3356     }
3357
3358     void compileGetCallee()
3359     {
3360         setJSValue(m_out.loadPtr(addressFor(JSStack::Callee)));
3361     }
3362
3363     void compileGetScope()
3364     {
3365         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope));
3366     }
3367
3368     void compileGetMyScope()
3369     {
3370         setJSValue(m_out.loadPtr(addressFor(
3371             m_node->origin.semantic.stackOffset() + JSStack::ScopeChain)));
3372     }
3373
3374     void compileSkipScope()
3375     {
3376         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
3377     }
3378
3379     void compileGetClosureRegisters()
3380     {
3381         if (WriteBarrierBase<Unknown>* registers = m_graph.tryGetRegisters(m_node->child1().node())) {
3382             setStorage(m_out.constIntPtr(registers));
3383             return;
3384         }
3385
3386         setStorage(m_out.loadPtr(
3387             lowCell(m_node->child1()), m_heaps.JSVariableObject_registers));
3388     }
3389
3390     void compileGetClosureVar()
3391     {
3392         setJSValue(m_out.load64(
3393             addressFor(lowStorage(m_node->child1()), m_node->varNumber())));
3394     }
3395
3396     void compilePutClosureVar()
3397     {
3398         m_out.store64(
3399             lowJSValue(m_node->child3()),
3400             addressFor(lowStorage(m_node->child2()), m_node->varNumber()));
3401     }
3402
3403     void compileCompareEq()
3404     {
3405         if (m_node->isBinaryUseKind(Int32Use)
3406             || m_node->isBinaryUseKind(Int52RepUse)
3407             || m_node->isBinaryUseKind(DoubleRepUse)
3408             || m_node->isBinaryUseKind(ObjectUse)
3409             || m_node->isBinaryUseKind(BooleanUse)
3410             || m_node->isBinaryUseKind(StringIdentUse)) {
3411             compileCompareStrictEq();
3412             return;
3413         }
3414
3415         if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) {
3416             compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1());
3417             return;
3418         }
3419
3420         if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) {
3421             compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2());
3422             return;
3423         }
3424
3425         if (m_node->isBinaryUseKind(UntypedUse)) {
3426             nonSpeculativeCompare(LLVMIntEQ, operationCompareEq);
3427             return;
3428         }
3429
3430         RELEASE_ASSERT_NOT_REACHED();
3431     }
3432
3433     void compileCompareEqConstant()
3434     {
3435         ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull());
3436         setBoolean(
3437             equalNullOrUndefined(
3438                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
3439     }
3440
3441     void compileCompareStrictEq()
3442     {
3443         if (m_node->isBinaryUseKind(Int32Use)) {
3444             setBoolean(
3445                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3446             return;
3447         }
3448
3449         if (m_node->isBinaryUseKind(Int52RepUse)) {
3450             Int52Kind kind;
3451             LValue left = lowWhicheverInt52(m_node->child1(), kind);
3452             LValue right = lowInt52(m_node->child2(), kind);
3453             setBoolean(m_out.equal(left, right));
3454             return;
3455         }
3456
3457         if (m_node->isBinaryUseKind(DoubleRepUse)) {
3458             setBoolean(
3459                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
3460             return;
3461         }
3462
3463         if (m_node->isBinaryUseKind(StringIdentUse)) {
3464             setBoolean(
3465                 m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2())));
3466             return;
3467         }
3468
3469         if (m_node->isBinaryUseKind(ObjectUse)) {
3470             setBoolean(
3471                 m_out.equal(
3472                     lowNonNullObject(m_node->child1()),
3473                     lowNonNullObject(m_node->child2())));
3474             return;
3475         }
3476
3477         if (m_node->isBinaryUseKind(BooleanUse)) {
3478             setBoolean(
3479                 m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2())));
3480             return;
3481         }
3482
3483         if (m_node->isBinaryUseKind(MiscUse, UntypedUse)
3484             || m_node->isBinaryUseKind(UntypedUse, MiscUse)) {
3485             speculate(m_node->child1());
3486             speculate(m_node->child2());
3487             LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation);
3488             LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation);
3489             setBoolean(m_out.equal(left, right));
3490             return;
3491         }
3492
3493         if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse)
3494             || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) {
3495             Edge leftEdge = m_node->childFor(StringIdentUse);
3496             Edge rightEdge = m_node->childFor(NotStringVarUse);
3497
3498             LValue left = lowStringIdent(leftEdge);
3499             LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation);
3500
3501             LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is cell case"));
3502             LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is string case"));
3503             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar continuation"));
3504
3505             ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
3506             m_out.branch(isCell(rightValue), unsure(isCellCase), unsure(continuation));
3507
3508             LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
3509             ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse);
3510             m_out.branch(isString(rightValue), unsure(isStringCase), unsure(continuation));
3511
3512             m_out.appendTo(isStringCase, continuation);
3513             LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value);
3514             speculateStringIdent(rightEdge, rightValue, right);
3515             ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right));
3516             m_out.jump(continuation);
3517
3518             m_out.appendTo(continuation, lastNext);
3519             setBoolean(m_out.phi(m_out.boolean, notCellResult, notStringResult, isStringResult));
3520             return;
3521         }
3522
3523         RELEASE_ASSERT_NOT_REACHED();
3524     }
3525
3526     void compileCompareStrictEqConstant()
3527     {
3528         JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node());
3529
3530         setBoolean(
3531             m_out.equal(
3532                 lowJSValue(m_node->child1()),
3533                 m_out.constInt64(JSValue::encode(constant))));
3534     }
3535
3536     void compileCompareLess()
3537     {
3538         compare(LLVMIntSLT, LLVMRealOLT, operationCompareLess);
3539     }
3540
3541     void compileCompareLessEq()
3542     {
3543         compare(LLVMIntSLE, LLVMRealOLE, operationCompareLessEq);
3544     }
3545
3546     void compileCompareGreater()
3547     {
3548         compare(LLVMIntSGT, LLVMRealOGT, operationCompareGreater);
3549     }
3550
3551     void compileCompareGreaterEq()
3552     {
3553         compare(LLVMIntSGE, LLVMRealOGE, operationCompareGreaterEq);
3554     }
3555
3556     void compileLogicalNot()
3557     {
3558         setBoolean(m_out.bitNot(boolify(m_node->child1())));
3559     }
3560
3561     void compileCallOrConstruct()
3562     {
3563         int dummyThisArgument = m_node->op() == Call ? 0 : 1;
3564         int numPassedArgs = m_node->numChildren() - 1;
3565         int numArgs = numPassedArgs + dummyThisArgument;
3566
3567         LValue callee = lowJSValue(m_graph.varArgChild(m_node, 0));
3568
3569         unsigned stackmapID = m_stackmapIDs++;
3570
3571         Vector<LValue> arguments;
3572         arguments.append(m_out.constInt64(stackmapID));
3573         arguments.append(m_out.constInt32(sizeOfCall()));
3574         arguments.append(constNull(m_out.ref8));
3575         arguments.append(m_out.constInt32(1 + JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + numArgs));
3576         arguments.append(callee); // callee -> %rax
3577         arguments.append(getUndef(m_out.int64)); // code block
3578         arguments.append(getUndef(m_out.int64)); // scope chain
3579         arguments.append(callee); // callee -> stack
3580         arguments.append(m_out.constInt64(numArgs)); // argument count and zeros for the tag
3581         if (dummyThisArgument)
3582             arguments.append(getUndef(m_out.int64));
3583         for (int i = 0; i < numPassedArgs; ++i)
3584             arguments.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
3585
3586         callPreflight();
3587
3588         LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments);
3589         setInstructionCallingConvention(call, LLVMWebKitJSCallConv);
3590
3591         m_ftlState.jsCalls.append(JSCall(stackmapID, m_node));
3592
3593         setJSValue(call);
3594     }
3595
3596     void compileJump()
3597     {
3598         m_out.jump(lowBlock(m_node->targetBlock()));
3599     }
3600
3601     void compileBranch()
3602     {
3603         m_out.branch(
3604             boolify(m_node->child1()),
3605             WeightedTarget(
3606                 lowBlock(m_node->branchData()->taken.block),
3607                 m_node->branchData()->taken.count),
3608             WeightedTarget(
3609                 lowBlock(m_node->branchData()->notTaken.block),
3610                 m_node->branchData()->notTaken.count));
3611     }
3612
3613     void compileSwitch()
3614     {
3615         SwitchData* data = m_node->switchData();
3616         switch (data->kind) {
3617         case SwitchImm: {
3618             Vector<ValueFromBlock, 2> intValues;
3619             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
3620
3621             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
3622
3623             switch (m_node->child1().useKind()) {
3624             case Int32Use: {
3625                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
3626                 m_out.jump(switchOnInts);
3627                 break;
3628             }
3629
3630             case UntypedUse: {
3631                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
3632                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
3633                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
3634
3635                 LValue boxedValue = lowJSValue(m_node->child1());
3636                 m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt));
3637
3638                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
3639
3640                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
3641                 m_out.jump(switchOnInts);
3642
3643                 m_out.appendTo(isNotInt, isDouble);
3644                 m_out.branch(
3645                     isCellOrMisc(boxedValue),
3646                     usually(lowBlock(data->fallThrough.block)), rarely(isDouble));
3647
3648                 m_out.appendTo(isDouble, innerLastNext);
3649                 LValue doubleValue = unboxDouble(boxedValue);
3650                 LValue intInDouble = m_out.fpToInt32(doubleValue);
3651                 intValues.append(m_out.anchor(intInDouble));
3652                 m_out.branch(
3653                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
3654                     unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block)));
3655                 break;
3656             }
3657
3658             default:
3659                 RELEASE_ASSERT_NOT_REACHED();
3660                 break;
3661             }
3662
3663             m_out.appendTo(switchOnInts, lastNext);
3664             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
3665             return;
3666         }
3667
3668         case SwitchChar: {
3669             LValue stringValue;
3670
3671             // FIXME: We should use something other than unsure() for the branch weight
3672             // of the fallThrough block. The main challenge is just that we have multiple
3673             // branches to fallThrough but a single count, so we would need to divvy it up
3674             // among the different lowered branches.
3675             // https://bugs.webkit.org/show_bug.cgi?id=129082
3676
3677             switch (m_node->child1().useKind()) {
3678             case StringUse: {
3679                 stringValue = lowString(m_node->child1());
3680                 break;
3681             }
3682
3683             case UntypedUse: {
3684                 LValue unboxedValue = lowJSValue(m_node->child1());
3685
3686                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
3687                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
3688
3689                 m_out.branch(
3690                     isNotCell(unboxedValue),
3691                     unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase));
3692
3693                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
3694                 LValue cellValue = unboxedValue;
3695                 m_out.branch(
3696                     isNotString(cellValue),
3697                     unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase));
3698
3699                 m_out.appendTo(isStringCase, lastNext);
3700                 stringValue = cellValue;
3701                 break;
3702             }
3703
3704             default:
3705                 RELEASE_ASSERT_NOT_REACHED();
3706                 break;
3707             }
3708
3709             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
3710             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
3711             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
3712             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
3713             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
3714             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
3715
3716             m_out.branch(
3717                 m_out.notEqual(
3718                     m_out.load32NonNegative(stringValue, m_heaps.JSString_length),
3719                     m_out.int32One),
3720                 unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1));
3721
3722             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
3723             Vector<ValueFromBlock, 2> values;
3724             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
3725             values.append(m_out.anchor(fastValue));
3726             m_out.branch(m_out.isNull(fastValue), rarely(needResolution), usually(resolved));
3727
3728             m_out.appendTo(needResolution, resolved);
3729             values.append(m_out.anchor(
3730                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
3731             m_out.jump(resolved);
3732
3733             m_out.appendTo(resolved, is8Bit);
3734             LValue value = m_out.phi(m_out.intPtr, values);
3735             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
3736             m_out.branch(
3737                 m_out.testNonZero32(
3738                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
3739                     m_out.constInt32(StringImpl::flagIs8Bit())),
3740                 unsure(is8Bit), unsure(is16Bit));
3741
3742             Vector<ValueFromBlock, 2> characters;
3743             m_out.appendTo(is8Bit, is16Bit);
3744             characters.append(m_out.anchor(
3745                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
3746             m_out.jump(continuation);
3747
3748             m_out.appendTo(is16Bit, continuation);
3749             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
3750             m_out.jump(continuation);
3751
3752             m_out.appendTo(continuation, lastNext);
3753             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
3754             return;
3755         }
3756
3757         case SwitchString:
3758             RELEASE_ASSERT_NOT_REACHED();
3759             break;
3760         }
3761
3762         RELEASE_ASSERT_NOT_REACHED();
3763     }
3764
3765     void compileReturn()
3766     {
3767         m_out.ret(lowJSValue(m_node->child1()));
3768     }
3769
3770     void compileForceOSRExit()
3771     {
3772         terminate(InadequateCoverage);
3773     }
3774
3775     void compileThrow()
3776     {
3777         terminate(Uncountable);
3778     }
3779
3780     void compileInvalidationPoint()
3781     {
3782         if (verboseCompilationEnabled())
3783             dataLog("    Invalidation point with availability: ", m_availability, "\n");
3784
3785         m_ftlState.jitCode->osrExit.append(OSRExit(
3786             UncountableInvalidation, InvalidValueFormat, MethodOfGettingAValueProfile(),
3787             m_codeOriginForExitTarget, m_codeOriginForExitProfile,
3788             m_availability.numberOfArguments(), m_availability.numberOfLocals()));
3789         m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
3790
3791         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
3792         OSRExitCompilationInfo& info = m_ftlState.finalizer->osrExit.last();
3793
3794         ExitArgumentList arguments;
3795
3796         buildExitArguments(exit, arguments, FormattedValue(), exit.m_codeOrigin);
3797         callStackmap(exit, arguments);
3798
3799         info.m_isInvalidationPoint = true;
3800     }
3801
3802     void compileCheckArgumentsNotCreated()
3803     {
3804         ASSERT(!isEmptySpeculation(
3805             m_state.variables().operand(
3806                 m_graph.argumentsRegisterFor(m_node->origin.semantic)).m_type));
3807
3808         checkArgumentsNotCreated();
3809     }
3810
3811     void compileIsUndefined()
3812     {
3813         setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
3814     }
3815
3816     void compileIsBoolean()
3817     {
3818         setBoolean(isBoolean(lowJSValue(m_node->child1())));
3819     }
3820
3821     void compileIsNumber()
3822     {
3823         setBoolean(isNumber(lowJSValue(m_node->child1())));
3824     }
3825
3826     void compileIsString()
3827     {
3828         LValue value = lowJSValue(m_node->child1());
3829
3830         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsString cell case"));
3831         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsString continuation"));
3832
3833         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
3834         m_out.branch(isCell(value), unsure(isCellCase), unsure(continuation));
3835
3836         LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
3837         ValueFromBlock cellResult = m_out.anchor(isString(value));
3838         m_out.jump(continuation);
3839
3840         m_out.appendTo(continuation, lastNext);
3841         setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult));
3842     }
3843
3844     void compileIsObject()
3845     {
3846         LValue pointerResult = vmCall(
3847             m_out.operation(operationIsObject), m_callFrame, lowJSValue(m_node->child1()));
3848         setBoolean(m_out.notNull(pointerResult));
3849     }
3850
3851     void compileIsFunction()
3852     {
3853         LValue pointerResult = vmCall(
3854             m_out.operation(operationIsFunction), lowJSValue(m_node->child1()));
3855         setBoolean(m_out.notNull(pointerResult));
3856     }
3857
3858     void compileCheckHasInstance()
3859     {
3860         speculate(
3861             Uncountable, noValue(), 0,
3862             m_out.testIsZero8(
3863                 m_out.load8(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags),
3864                 m_out.constInt8(ImplementsDefaultHasInstance)));
3865     }
3866
3867     void compileInstanceOf()
3868     {
3869         LValue cell;
3870
3871         if (m_node->child1().useKind() == UntypedUse)
3872             cell = lowJSValue(m_node->child1());
3873         else
3874             cell = lowCell(m_node->child1());
3875
3876         LValue prototype = lowCell(m_node->child2());
3877
3878         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("InstanceOf cell case"));
3879         LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("InstanceOf loop"));
3880         LBasicBlock notYetInstance = FTL_NEW_BLOCK(m_out, ("InstanceOf not yet instance"));
3881         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("InstanceOf continuation"));
3882
3883         LValue condition;
3884         if (m_node->child1().useKind() == UntypedUse)
3885             condition = isCell(cell);
3886         else
3887             condition = m_out.booleanTrue;
3888
3889         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
3890         m_out.branch(condition, unsure(isCellCase), unsure(continuation));
3891
3892         LBasicBlock lastNext = m_out.appendTo(isCellCase, loop);
3893
3894         speculate(BadType, noValue(), 0, isNotObject(prototype));
3895
3896         ValueFromBlock originalValue = m_out.anchor(cell);
3897         m_out.jump(loop);
3898
3899         m_out.appendTo(loop, notYetInstance);
3900         LValue value = m_out.phi(m_out.int64, originalValue);
3901         LValue structure = loadStructure(value);
3902         LValue currentPrototype = m_out.load64(structure, m_heaps.Structure_prototype);
3903         ValueFromBlock isInstanceResult = m_out.anchor(m_out.booleanTrue);
3904         m_out.branch(
3905             m_out.equal(currentPrototype, prototype),
3906             unsure(continuation), unsure(notYetInstance));
3907
3908         m_out.appendTo(notYetInstance, continuation);
3909         ValueFromBlock notInstanceResult = m_out.anchor(m_out.booleanFalse);
3910         addIncoming(value, m_out.anchor(currentPrototype));
3911         m_out.branch(isCell(currentPrototype), unsure(loop), unsure(continuation));
3912
3913         m_out.appendTo(continuation, lastNext);
3914         setBoolean(
3915             m_out.phi(m_out.boolean, notCellResult, isInstanceResult, notInstanceResult));
3916     }
3917
3918     void compileCountExecution()
3919     {
3920         TypedPointer counter = m_out.absolute(m_node->executionCounter()->address());
3921         m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter);
3922     }
3923
3924     void compileStoreBarrier()
3925     {
3926         emitStoreBarrier(lowCell(m_node->child1()));
3927     }
3928
3929     void compileStoreBarrierWithNullCheck()
3930     {
3931 #if ENABLE(GGC)
3932         LBasicBlock isNotNull = FTL_NEW_BLOCK(m_out, ("Store barrier with null check value not null"));
3933         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation"));
3934
3935         LValue base = lowJSValue(m_node->child1());
3936         m_out.branch(m_out.isZero64(base), unsure(continuation), unsure(isNotNull));
3937         LBasicBlock lastNext = m_out.appendTo(isNotNull, continuation);
3938         emitStoreBarrier(base);
3939         m_out.appendTo(continuation, lastNext);
3940 #else
3941         speculate(m_node->child1());
3942 #endif
3943     }
3944
3945     LValue didOverflowStack()
3946     {
3947         // This does a very simple leaf function analysis. The invariant of FTL call
3948         // frames is that the caller had already done enough of a stack check to
3949         // prove that this call frame has enough stack to run, and also enough stack
3950         // to make runtime calls. So, we only need to stack check when making calls
3951         // to other JS functions. If we don't find such calls then we don't need to
3952         // do any stack checks.
3953
3954         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
3955             BasicBlock* block = m_graph.block(blockIndex);
3956             if (!block)
3957                 continue;
3958
3959             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
3960                 Node* node = block->at(nodeIndex);
3961
3962                 switch (node->op()) {
3963                 case GetById:
3964                 case PutById:
3965                 case Call:
3966                 case Construct:
3967                     return m_out.below(
3968                         m_callFrame,
3969                         m_out.loadPtr(
3970                             m_out.absolute(vm().addressOfFTLStackLimit())));
3971
3972                 default:
3973                     break;
3974                 }
3975             }
3976         }
3977
3978         return m_out.booleanFalse;
3979     }
3980
3981     LValue numberOrNotCellToInt32(Edge edge, LValue value)
3982     {
3983         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 int case"));
3984         LBasicBlock notIntCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not int case"));
3985         LBasicBlock doubleCase = 0;
3986         LBasicBlock notNumberCase = 0;
3987         if (edge.useKind() == NotCellUse) {
3988             doubleCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 double case"));
3989             notNumberCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not number case"));
3990         }
3991         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ValueToInt32 continuation"));
3992
3993         Vector<ValueFromBlock> results;
3994
3995         m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase));
3996
3997         LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase);
3998         results.append(m_out.anchor(unboxInt32(value)));
3999         m_out.jump(continuation);
4000
4001         if (edge.useKind() == NumberUse) {
4002             m_out.appendTo(notIntCase, continuation);
4003             FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value));
4004             results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
4005             m_out.jump(continuation);
4006         } else {
4007             m_out.appendTo(notIntCase, doubleCase);
4008             m_out.branch(isCellOrMisc(value), unsure(notNumberCase), unsure(doubleCase));
4009
4010             m_out.appendTo(doubleCase, notNumberCase);
4011             results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
4012             m_out.jump(continuation);
4013
4014             m_out.appendTo(notNumberCase, continuation);
4015
4016             FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCell, isCell(value));
4017
4018             LValue specialResult = m_out.select(
4019                 m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))),
4020                 m_out.int32One, m_out.int32Zero);
4021             results.append(m_out.anchor(specialResult));
4022             m_out.jump(continuation);
4023         }
4024
4025         m_out.appendTo(continuation, lastNext);
4026         return m_out.phi(m_out.int32, results);
4027     }
4028
4029     LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset)
4030     {
4031         return m_out.load64(addressOfProperty(storage, identifierNumber, offset));
4032     }
4033
4034     void storeProperty(
4035         LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset)
4036     {
4037         m_out.store64(value, addressOfProperty(storage, identifierNumber, offset));
4038     }
4039
4040     TypedPointer addressOfProperty(
4041         LValue storage, unsigned identifierNumber, PropertyOffset offset)
4042     {
4043         return m_out.address(
4044             m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset));
4045     }
4046
4047     LValue storageForTransition(
4048         LValue object, PropertyOffset offset,
4049         Structure* previousStructure, Structure* nextStructure)
4050     {
4051         if (isInlineOffset(offset))
4052             return object;
4053
4054         if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity())
4055             return m_out.loadPtr(object, m_heaps.JSObject_butterfly);
4056
4057         LValue result;
4058         if (!previousStructure->outOfLineCapacity())
4059             result = allocatePropertyStorage(object, previousStructure);
4060         else {
4061             result = reallocatePropertyStorage(
4062                 object, m_out.loadPtr(object, m_heaps.JSObject_butterfly),
4063                 previousStructure, nextStructure);
4064         }
4065
4066         emitStoreBarrier(object);
4067
4068         return result;
4069     }
4070
4071     LValue allocatePropertyStorage(LValue object, Structure* previousStructure)
4072     {
4073         if (previousStructure->couldHaveIndexingHeader()) {
4074             return vmCall(
4075                 m_out.operation(
4076                     operationReallocateButterflyToHavePropertyStorageWithInitialCapacity),
4077                 m_callFrame, object);
4078         }
4079
4080         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorage slow path"));
4081         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorage continuation"));
4082
4083         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
4084
4085         LValue endOfStorage = allocateBasicStorageAndGetEnd(
4086             m_out.constIntPtr(initialOutOfLineCapacity * sizeof(JSValue)), slowPath);
4087
4088         ValueFromBlock fastButterfly = m_out.anchor(
4089             m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage));
4090
4091         m_out.jump(continuation);
4092
4093         m_out.appendTo(slowPath, continuation);
4094
4095         ValueFromBlock slowButterfly = m_out.anchor(vmCall(
4096             m_out.operation(operationAllocatePropertyStorageWithInitialCapacity), m_callFrame));
4097
4098         m_out.jump(continuation);
4099
4100         m_out.appendTo(continuation, lastNext);
4101
4102         LValue result = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
4103         m_out.storePtr(result, object, m_heaps.JSObject_butterfly);
4104
4105         return result;
4106     }
4107
4108     LValue reallocatePropertyStorage(
4109         LValue object, LValue oldStorage, Structure* previous, Structure* next)
4110     {
4111         size_t oldSize = previous->outOfLineCapacity() * sizeof(JSValue);
4112         size_t newSize = oldSize * outOfLineGrowthFactor;
4113
4114         ASSERT_UNUSED(next, newSize == next->outOfLineCapacity() * sizeof(JSValue));
4115
4116         if (previous->couldHaveIndexingHeader()) {
4117             LValue newAllocSize = m_out.constInt64(newSize / sizeof(JSValue));
4118             return vmCall(m_out.operation(operationReallocateButterflyToGrowPropertyStorage), m_callFrame, object, newAllocSize);
4119         }
4120
4121         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("reallocatePropertyStorage slow path"));
4122         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("reallocatePropertyStorage continuation"));
4123         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
4124
4125         LValue endOfStorage =
4126             allocateBasicStorageAndGetEnd(m_out.constIntPtr(newSize), slowPath);
4127
4128         ValueFromBlock fastButterfly = m_out.anchor(m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage));
4129
4130         m_out.jump(continuation);
4131
4132         m_out.appendTo(slowPath, continuation);
4133
4134         LValue newAllocSize = m_out.constInt64(newSize / sizeof(JSValue));
4135
4136         LValue storageLocation = vmCall(m_out.operation(operationAllocatePropertyStorage), m_callFrame, newAllocSize);
4137
4138         ValueFromBlock slowButterfly = m_out.anchor(storageLocation);
4139
4140         m_out.jump(continuation);
4141
4142         m_out.appendTo(continuation, lastNext);
4143
4144         LValue result = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
4145
4146         ptrdiff_t headerSize = -sizeof(JSValue) - sizeof(void *);
4147         ptrdiff_t endStorage = headerSize - static_cast<ptrdiff_t>(oldSize);
4148
4149         for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) {
4150             LValue loaded =
4151                 m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset));
4152             m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset));
4153         }
4154
4155         m_out.storePtr(result, m_out.address(object, m_heaps.JSObject_butterfly));
4156
4157         return result;
4158     }
4159
4160     LValue getById(LValue base)
4161     {
4162         StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
4163
4164         // Arguments: id, bytes, target, numArgs, args...
4165         unsigned stackmapID = m_stackmapIDs++;
4166
4167         if (Options::verboseCompilation())
4168             dataLog("    Emitting GetById patchpoint with stackmap #", stackmapID, "\n");
4169
4170         LValue call = m_out.call(
4171             m_out.patchpointInt64Intrinsic(),
4172             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfGetById()),
4173             constNull(m_out.ref8), m_out.constInt32(1), base);
4174         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
4175
4176         m_ftlState.getByIds.append(GetByIdDescriptor(stackmapID, m_node->origin.semantic, uid));
4177
4178         return call;
4179     }
4180
4181     TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge)
4182     {
4183         return m_out.baseIndex(
4184             heap, storage, m_out.zeroExt(index, m_out.intPtr),
4185             m_state.forNode(edge).m_value);
4186     }
4187
4188     void compare(
4189         LIntPredicate intCondition, LRealPredicate realCondition,
4190         S_JITOperation_EJJ helperFunction)
4191     {
4192         if (m_node->isBinaryUseKind(Int32Use)) {
4193             LValue left = lowInt32(m_node->child1());
4194             LValue right = lowInt32(m_node->child2());
4195             setBoolean(m_out.icmp(intCondition, left, right));
4196             return;
4197         }
4198
4199         if (m_node->isBinaryUseKind(Int52RepUse)) {
4200             Int52Kind kind;
4201             LValue left = lowWhicheverInt52(m_node->child1(), kind);
4202             LValue right = lowInt52(m_node->child2(), kind);
4203             setBoolean(m_out.icmp(intCondition, left, right));
4204             return;
4205         }
4206
4207         if (m_node->isBinaryUseKind(DoubleRepUse)) {
4208             LValue left = lowDouble(m_node->child1());
4209             LValue right = lowDouble(m_node->child2());
4210             setBoolean(m_out.fcmp(realCondition, left, right));
4211             return;
4212         }
4213
4214         if (m_node->isBinaryUseKind(UntypedUse)) {
4215             nonSpeculativeCompare(intCondition, helperFunction);
4216             return;
4217         }
4218
4219         RELEASE_ASSERT_NOT_REACHED();
4220     }
4221
4222     void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild)
4223     {
4224         LValue rightCell = lowCell(rightChild);
4225         LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation);
4226
4227         speculateTruthyObject(rightChild, rightCell, SpecObject);
4228
4229         LBasicBlock leftCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left cell case"));
4230         LBasicBlock leftNotCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left not cell case"));
4231         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject continuation"));
4232
4233         m_out.branch(isCell(leftValue), unsure(leftCellCase), unsure(leftNotCellCase));
4234
4235         LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase);
4236         speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCell));
4237         ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue));
4238         m_out.jump(continuation);
4239
4240         m_out.appendTo(leftNotCellCase, continuation);
4241         FTL_TYPE_CHECK(
4242             jsValueValue(leftValue), leftChild, SpecOther | SpecCell, isNotOther(leftValue));
4243         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4244         m_out.jump(continuation);
4245
4246         m_out.appendTo(continuation, lastNext);
4247         setBoolean(m_out.phi(m_out.boolean, cellResult, notCellResult));
4248     }
4249
4250     void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter)
4251     {
4252         if (masqueradesAsUndefinedWatchpointIsStillValid()) {
4253             FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
4254             return;
4255         }
4256
4257         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
4258         FTL_TYPE_CHECK(
4259             jsValueValue(cell), edge, filter,
4260             m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())));
4261         speculate(
4262             BadType, jsValueValue(cell), edge.node(),
4263             m_out.testNonZero8(
4264                 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
4265                 m_out.constInt8(MasqueradesAsUndefined)));
4266     }
4267
4268     void nonSpeculativeCompare(LIntPredicate intCondition, S_JITOperation_EJJ helperFunction)
4269     {
4270         LValue left = lowJSValue(m_node->child1());
4271         LValue right = lowJSValue(m_node->child2());
4272
4273         LBasicBlock leftIsInt = FTL_NEW_BLOCK(m_out, ("CompareEq untyped left is int"));
4274         LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped fast path"));
4275         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped slow path"));
4276         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEq untyped continuation"));
4277
4278         m_out.branch(isNotInt32(left), rarely(slowPath), usually(leftIsInt));
4279
4280         LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath);
4281         m_out.branch(isNotInt32(right), rarely(slowPath), usually(fastPath));
4282
4283         m_out.appendTo(fastPath, slowPath);
4284         ValueFromBlock fastResult = m_out.anchor(
4285             m_out.icmp(intCondition, unboxInt32(left), unboxInt32(right)));
4286         m_out.jump(continuation);
4287
4288         m_out.appendTo(slowPath, continuation);
4289         ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
4290             m_out.operation(helperFunction), m_callFrame, left, right)));
4291         m_out.jump(continuation);
4292
4293         m_out.appendTo(continuation, lastNext);
4294         setBoolean(m_out.phi(m_out.boolean, fastResult, slowResult));
4295     }
4296
4297     LValue allocateCell(LValue allocator, Structure* structure, LBasicBlock slowPath)
4298     {
4299         LBasicBlock success = FTL_NEW_BLOCK(m_out, ("object allocation success"));
4300
4301         LValue result = m_out.loadPtr(
4302             allocator, m_heaps.MarkedAllocator_freeListHead);
4303
4304         m_out.branch(m_out.notNull(result), usually(success), rarely(slowPath));
4305
4306         m_out.appendTo(success);
4307
4308         m_out.storePtr(
4309             m_out.loadPtr(result, m_heaps.JSCell_freeListNext),
4310             allocator, m_heaps.MarkedAllocator_freeListHead);
4311
4312         m_out.store32(m_out.constInt32(structure->id()), result, m_heaps.JSCell_structureID);
4313         m_out.store8(m_out.constInt8(structure->indexingType()), result, m_heaps.JSCell_indexingType);
4314         m_out.store8(m_out.constInt8(structure->typeInfo().type()), result, m_heaps.JSCell_typeInfoType);
4315         m_out.store8(m_out.constInt8(structure->typeInfo().inlineTypeFlags()), result, m_heaps.JSCell_typeInfoFlags);
4316         m_out.store8(m_out.constInt8(JSCell::NotMarked), result, m_heaps.JSCell_gcData);
4317
4318         return result;
4319     }
4320
4321     LValue allocateObject(
4322         LValue allocator, Structure* structure, LValue butterfly, LBasicBlock slowPath)
4323     {
4324         LValue result = allocateCell(allocator, structure, slowPath);
4325         m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly);
4326         return result;
4327     }
4328
4329     template<typename ClassType>
4330     LValue allocateObject(Structure* structure, LValue butterfly, LBasicBlock slowPath)
4331     {
4332         MarkedAllocator* allocator;
4333         size_t size = ClassType::allocationSize(0);
4334         if (ClassType::needsDestruction && ClassType::hasImmortalStructure)
4335             allocator = &vm().heap.allocatorForObjectWithImmortalStructureDestructor(size);
4336         else if (ClassType::needsDestruction)
4337             allocator = &vm().heap.allocatorForObjectWithNormalDestructor(size);
4338         else
4339             allocator = &vm().heap.allocatorForObjectWithoutDestructor(size);
4340         return allocateObject(m_out.constIntPtr(allocator), structure, butterfly, slowPath);
4341     }
4342
4343     // Returns a pointer to the end of the allocation.
4344     LValue allocateBasicStorageAndGetEnd(LValue size, LBasicBlock slowPath)
4345     {
4346         CopiedAllocator& allocator = vm().heap.storageAllocator();
4347
4348         LBasicBlock success = FTL_NEW_BLOCK(m_out, ("storage allocation success"));
4349
4350         LValue remaining = m_out.loadPtr(m_out.absolute(&allocator.m_currentRemaining));
4351         LValue newRemaining = m_out.sub(remaining, size);
4352
4353         m_out.branch(
4354             m_out.lessThan(newRemaining, m_out.intPtrZero),
4355             rarely(slowPath), usually(success));
4356
4357         m_out.appendTo(success);
4358
4359         m_out.storePtr(newRemaining, m_out.absolute(&allocator.m_currentRemaining));
4360         return m_out.sub(
4361             m_out.loadPtr(m_out.absolute(&allocator.m_currentPayloadEnd)), newRemaining);
4362     }
4363
4364     struct ArrayValues {
4365         ArrayValues()
4366             : array(0)
4367             , butterfly(0)
4368         {
4369         }
4370
4371         ArrayValues(LValue array, LValue butterfly)
4372             : array(array)
4373             , butterfly(butterfly)
4374         {
4375         }
4376
4377         LValue array;
4378         LValue butterfly;
4379     };
4380     ArrayValues allocateJSArray(
4381         Structure* structure, unsigned numElements, LBasicBlock slowPath)
4382     {
4383         ASSERT(
4384             hasUndecided(structure->indexingType())
4385             || hasInt32(structure->indexingType())
4386             || hasDouble(structure->indexingType())
4387             || hasContiguous(structure->indexingType()));
4388
4389         unsigned vectorLength = std::max(BASE_VECTOR_LEN, numElements);
4390
4391         LValue endOfStorage = allocateBasicStorageAndGetEnd(
4392             m_out.constIntPtr(sizeof(JSValue) * vectorLength + sizeof(IndexingHeader)),
4393             slowPath);
4394
4395         LValue butterfly = m_out.sub(
4396             endOfStorage, m_out.constIntPtr(sizeof(JSValue) * vectorLength));
4397
4398         LValue object = allocateObject<JSArray>(
4399             structure, butterfly, slowPath);
4400
4401         m_out.store32(m_out.constInt32(numElements), butterfly, m_heaps.Butterfly_publicLength);
4402         m_out.store32(m_out.constInt32(vectorLength), butterfly, m_heaps.Butterfly_vectorLength);
4403
4404         if (hasDouble(structure->indexingType())) {
4405             for (unsigned i = numElements; i < vectorLength; ++i) {
4406                 m_out.store64(
4407                     m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
4408                     butterfly, m_heaps.indexedDoubleProperties[i]);
4409             }
4410         }
4411
4412         return ArrayValues(object, butterfly);
4413     }
4414
4415     ArrayValues allocateJSArray(Structure* structure, unsigned numElements)
4416     {
4417         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("JSArray allocation slow path"));
4418         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("JSArray allocation continuation"));
4419
4420         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
4421
4422         ArrayValues fastValues = allocateJSArray(structure, numElements, slowPath);
4423         ValueFromBlock fastArray = m_out.anchor(fastValues.array);
4424         ValueFromBlock fastButterfly = m_out.anchor(fastValues.butterfly);
4425
4426         m_out.jump(continuation);
4427
4428         m_out.appendTo(slowPath, continuation);
4429
4430         ValueFromBlock slowArray = m_out.anchor(vmCall(
4431             m_out.operation(operationNewArrayWithSize), m_callFrame,
4432             m_out.constIntPtr(structure), m_out.constInt32(numElements)));
4433         ValueFromBlock slowButterfly = m_out.anchor(
4434             m_out.loadPtr(slowArray.value(), m_heaps.JSObject_butterfly));
4435
4436         m_out.jump(continuation);
4437
4438         m_out.appendTo(continuation, lastNext);
4439
4440         return ArrayValues(
4441             m_out.phi(m_out.intPtr, fastArray, slowArray),
4442             m_out.phi(m_out.intPtr, fastButterfly, slowButterfly));
4443     }
4444
4445     LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode, LValue base)
4446     {
4447         if (JSArrayBufferView* view = m_graph.tryGetFoldableView(baseEdge.node(), arrayMode))
4448             return m_out.constInt32(view->length());
4449         return m_out.load32NonNegative(base, m_heaps.JSArrayBufferView_length);
4450     }
4451
4452     LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode)
4453     {
4454         return typedArrayLength(baseEdge, arrayMode, lowCell(baseEdge));
4455     }
4456
4457     LValue boolify(Edge edge)
4458     {
4459         switch (edge.useKind()) {
4460         case BooleanUse:
4461             return lowBoolean(m_node->child1());
4462         case Int32Use:
4463             return m_out.notZero32(lowInt32(m_node->child1()));
4464         case DoubleRepUse:
4465             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
4466         case ObjectOrOtherUse:
4467             return m_out.bitNot(
4468                 equalNullOrUndefined(
4469                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
4470                     ManualOperandSpeculation));
4471         case StringUse: {
4472             LValue stringValue = lowString(m_node->child1());
4473             LValue length = m_out.load32NonNegative(stringValue, m_heaps.JSString_length);
4474             return m_out.notEqual(length, m_out.int32Zero);
4475         }
4476         case UntypedUse: {
4477             LValue value = lowJSValue(m_node->child1());
4478
4479             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped slow case"));
4480             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped fast case"));
4481             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Boolify untyped continuation"));
4482
4483             m_out.branch(isNotBoolean(value), rarely(slowCase), usually(fastCase));
4484
4485             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
4486             ValueFromBlock fastResult = m_out.anchor(unboxBoolean(value));
4487             m_out.jump(continuation);
4488
4489             m_out.appendTo(slowCase, continuation);
4490             ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
4491                 m_out.operation(operationConvertJSValueToBoolean), m_callFrame, value)));
4492             m_out.jump(continuation);
4493
4494             m_out.appendTo(continuation, lastNext);
4495             return m_out.phi(m_out.boolean, fastResult, slowResult);
4496         }
4497         default:
4498             RELEASE_ASSERT_NOT_REACHED();
4499             return 0;
4500         }
4501     }
4502
4503     enum StringOrObjectMode {
4504         AllCellsAreFalse,
4505         CellCaseSpeculatesObject
4506     };
4507     enum EqualNullOrUndefinedMode {
4508         EqualNull,
4509         EqualUndefined,
4510         EqualNullOrUndefined,
4511         SpeculateNullOrUndefined
4512     };
4513     LValue equalNullOrUndefined(
4514         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
4515         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
4516     {
4517         bool validWatchpoint = masqueradesAsUndefinedWatchpointIsStillValid();
4518
4519         LValue value = lowJSValue(edge, operandMode);
4520
4521         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
4522         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
4523         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
4524
4525         m_out.branch(isNotCell(value), unsure(primitiveCase), unsure(cellCase));
4526
4527         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
4528
4529         Vector<ValueFromBlock, 3> results;
4530
4531         switch (cellMode) {
4532         case AllCellsAreFalse:
4533             break;
4534         case CellCaseSpeculatesObject:
4535             FTL_TYPE_CHECK(
4536                 jsValueValue(value), edge, (~SpecCell) | SpecObject,
4537                 m_out.equal(
4538                     m_out.load32(value, m_heaps.JSCell_structureID),
4539                     m_out.constInt32(vm().stringStructure->id())));
4540             break;
4541         }
4542
4543         if (validWatchpoint) {
4544             results.append(m_out.anchor(m_out.booleanFalse));
4545             m_out.jump(continuation);
4546         } else {
4547             LBasicBlock masqueradesCase =
4548                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
4549
4550             results.append(m_out.anchor(m_out.booleanFalse));
4551
4552             m_out.branch(
4553                 m_out.testNonZero8(
4554                     m_out.load8(value, m_heaps.JSCell_typeInfoFlags),
4555                     m_out.constInt8(MasqueradesAsUndefined)),
4556                 rarely(masqueradesCase), usually(continuation));
4557
4558             m_out.appendTo(masqueradesCase, primitiveCase);
4559
4560             LValue structure = loadStructure(value);
4561
4562             results.append(m_out.anchor(
4563                 m_out.equal(
4564                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)),
4565                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
4566             m_out.jump(continuation);
4567         }
4568
4569         m_out.appendTo(primitiveCase, continuation);
4570
4571         LValue primitiveResult;
4572         switch (primitiveMode) {
4573         case EqualNull:
4574             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
4575             break;
4576         case EqualUndefined:
4577             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
4578             break;
4579         case EqualNullOrUndefined:
4580             primitiveResult = isOther(value);
4581             break;
4582         case SpeculateNullOrUndefined:
4583             FTL_TYPE_CHECK(
4584                 jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
4585             primitiveResult = m_out.booleanTrue;
4586             break;
4587         }
4588         results.append(m_out.anchor(primitiveResult));
4589         m_out.jump(continuation);
4590
4591         m_out.appendTo(continuation, lastNext);
4592
4593         return m_out.phi(m_out.boolean, results);
4594     }
4595
4596     template<typename FunctionType>
4597     void contiguousPutByValOutOfBounds(
4598         FunctionType slowPathFunction, LValue base, LValue storage, LValue index, LValue value,
4599         LBasicBlock continuation)
4600     {
4601         LValue isNotInBounds = m_out.aboveOrEqual(
4602             index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength));
4603         if (!m_node->arrayMode().isInBounds()) {
4604             LBasicBlock notInBoundsCase =
4605                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
4606             LBasicBlock performStore =
4607                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
4608
4609             m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore));
4610
4611             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
4612
4613             LValue isOutOfBounds = m_out.aboveOrEqual(
4614                 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength));
4615
4616             if (!m_node->arrayMode().isOutOfBounds())
4617                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
4618             else {
4619                 LBasicBlock outOfBoundsCase =
4620                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
4621                 LBasicBlock holeCase =
4622                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
4623
4624                 m_out.branch(isOutOfBounds, unsure(outOfBoundsCase), unsure(holeCase));
4625
4626                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
4627
4628                 vmCall(
4629                     m_out.operation(slowPathFunction),
4630                     m_callFrame, base, index, value);
4631
4632                 m_out.jump(continuation);
4633
4634                 m_out.appendTo(holeCase, innerLastNext);
4635             }
4636
4637             m_out.store32(
4638                 m_out.add(index, m_out.int32One),
4639                 storage, m_heaps.Butterfly_publicLength);
4640
4641             m_out.jump(performStore);
4642             m_out.appendTo(performStore, lastNext);
4643         }
4644     }
4645
4646     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
4647     {
4648         Vector<SwitchCase> cases;
4649         for (unsigned i = 0; i < data->cases.size(); ++i) {
4650             cases.append(SwitchCase(
4651                 constInt(type, data->cases[i].value.switchLookupValue()),
4652                 lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count)));
4653         }
4654
4655         m_out.switchInstruction(
4656             switchValue, cases,
4657             lowBlock(data->fallThrough.block), Weight(data->fallThrough.count));
4658     }
4659
4660     LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
4661     {
4662         LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough"));
4663         LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange"));
4664         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath"));
4665         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("doubleToInt32 continuation"));
4666
4667         Vector<ValueFromBlock, 2> results;
4668
4669         m_out.branch(
4670             m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)),
4671             unsure(greatEnough), unsure(slowPath));
4672
4673         LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
4674         m_out.branch(
4675             m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)),
4676             unsure(withinRange), unsure(slowPath));
4677
4678         m_out.appendTo(withinRange, slowPath);
4679         LValue fastResult;
4680         if (isSigned)
4681             fastResult = m_out.fpToInt32(doubleValue);
4682         else
4683             fastResult = m_out.fpToUInt32(doubleValue);
4684         results.append(m_out.anchor(fastResult));
4685         m_out.jump(continuation);
4686
4687         m_out.appendTo(slowPath, continuation);
4688         results.append(m_out.anchor(m_out.call(m_out.operation(toInt32), doubleValue)));
4689         m_out.jump(continuation);
4690
4691         m_out.appendTo(continuation, lastNext);
4692         return m_out.phi(m_out.int32, results);
4693     }
4694
4695     LValue doubleToInt32(LValue doubleValue)
4696     {
4697         if (Output::hasSensibleDoubleToInt())
4698             return sensibleDoubleToInt32(doubleValue);
4699
4700         double limit = pow(2, 31) - 1;
4701         return doubleToInt32(doubleValue, -limit, limit);
4702     }
4703
4704     LValue sensibleDoubleToInt32(LValue doubleValue)
4705     {
4706         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 slow path"));
4707         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 continuation"));
4708
4709         ValueFromBlock fastResult = m_out.anchor(
4710             m_out.sensibleDoubleToInt(doubleValue));
4711         m_out.branch(
4712             m_out.equal(fastResult.value(), m_out.constInt32(0x80000000)),
4713             rarely(slowPath), usually(continuation));
4714
4715         LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
4716         ValueFromBlock slowResult = m_out.anchor(
4717             m_out.call(m_out.operation(toInt32), doubleValue));
4718         m_out.jump(continuation);
4719
4720         m_out.appendTo(continuation, lastNext);
4721         return m_out.phi(m_out.int32, fastResult, slowResult);
4722     }
4723
4724     void checkArgumentsNotCreated()
4725     {
4726         CodeOrigin codeOrigin = m_node->origin.semantic;
4727         VirtualRegister argumentsRegister = m_graph.argumentsRegisterFor(codeOrigin);
4728         if (isEmptySpeculation(m_state.variables().operand(argumentsRegister).m_type))
4729             return;
4730
4731         VirtualRegister argsReg = m_graph.machineArgumentsRegisterFor(codeOrigin);
4732         speculate(
4733             ArgumentsEscaped, noValue(), 0,
4734             m_out.notZero64(m_out.load64(addressFor(argsReg))));
4735     }
4736
4737     void speculate(
4738         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
4739     {
4740         appendOSRExit(kind, lowValue, highValue, failCondition);
4741     }
4742
4743     void terminate(ExitKind kind)
4744     {
4745         speculate(kind, noValue(), 0, m_out.booleanTrue);
4746     }
4747
4748     void typeCheck(
4749         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
4750         LValue failCondition)
4751     {
4752         appendTypeCheck(lowValue, highValue, typesPassedThrough, failCondition);
4753     }
4754
4755     void appendTypeCheck(
4756         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
4757         LValue failCondition)
4758     {
4759         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
4760             return;
4761         ASSERT(mayHaveTypeCheck(highValue.useKind()));
4762         appendOSRExit(BadType, lowValue, highValue.node(), failCondition);
4763         m_interpreter.filter(highValue, typesPassedThrough);
4764     }
4765
4766     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4767     {
4768         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
4769
4770         if (edge->hasConstant()) {
4771             JSValue value = m_graph.valueOfJSConstant(edge.node());
4772             if (!value.isInt32()) {
4773                 terminate(Uncountable);
4774                 return m_out.int32Zero;
4775             }
4776             return m_out.constInt32(value.asInt32());
4777         }
4778
4779         LoweredNodeValue value = m_int32Values.get(edge.node());
4780         if (isValid(value))
4781             return value.value();
4782
4783         value = m_strictInt52Values.get(edge.node());
4784         if (isValid(value))
4785             return strictInt52ToInt32(edge, value.value());
4786
4787         value = m_int52Values.get(edge.node());
4788         if (isValid(value))
4789             return strictInt52ToInt32(edge, int52ToStrictInt52(value.value()));
4790
4791         value = m_jsValueValues.get(edge.node());
4792         if (isValid(value)) {
4793             LValue boxedResult = value.value();
4794             FTL_TYPE_CHECK(
4795                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
4796             LValue result = unboxInt32(boxedResult);
4797             setInt32(edge.node(), result);
4798             return result;
4799         }
4800
4801         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32));
4802         terminate(Uncountable);
4803         return m_out.int32Zero;
4804     }
4805
4806     enum Int52Kind { StrictInt52, Int52 };
4807     LValue lowInt52(Edge edge, Int52Kind kind)
4808     {
4809         RELEASE_ASSERT(edge.useKind() == Int52RepUse);
4810
4811         LoweredNodeValue value;
4812
4813         switch (kind) {
4814         case Int52:
4815             value = m_int52Values.get(edge.node());
4816             if (isValid(value))
4817                 return value.value();
4818
4819             value = m_strictInt52Values.get(edge.node());
4820             if (isValid(value))
4821                 return strictInt52ToInt52(value.value());
4822             break;
4823
4824         case StrictInt52:
4825             value = m_strictInt52Values.get(edge.node());
4826             if (isValid(value))
4827                 return value.value();
4828
4829             value = m_int52Values.get(edge.node());
4830             if (isValid(value))
4831                 return int52ToStrictInt52(value.value());
4832             break;
4833         }
4834
4835         RELEASE_ASSERT(!m_state.forNode(edge).m_type);
4836         terminate(Uncountable);
4837         return m_out.int64Zero;
4838     }
4839
4840     LValue lowInt52(Edge edge)
4841     {
4842         return lowInt52(edge, Int52);
4843     }
4844
4845     LValue lowStrictInt52(Edge edge)
4846     {
4847         return lowInt52(edge, StrictInt52);
4848     }
4849
4850     bool betterUseStrictInt52(Node* node)
4851     {
4852         return !isValid(m_int52Values.get(node));
4853     }
4854     bool betterUseStrictInt52(Edge edge)
4855     {
4856         return betterUseStrictInt52(edge.node());
4857     }
4858     template<typename T>
4859     Int52Kind bestInt52Kind(T node)
4860     {
4861         return betterUseStrictInt52(node) ? StrictInt52 : Int52;
4862     }
4863     Int52Kind opposite(Int52Kind kind)
4864     {
4865         switch (kind) {
4866         case Int52:
4867             return StrictInt52;
4868         case StrictInt52:
4869             return Int52;
4870         }
4871         RELEASE_ASSERT_NOT_REACHED();
4872         return Int52;
4873     }
4874
4875     LValue lowWhicheverInt52(Edge edge, Int52Kind& kind)
4876     {
4877         kind = bestInt52Kind(edge);
4878         return lowInt52(edge, kind);
4879     }
4880
4881     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4882     {
4883         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind()));
4884
4885         if (edge->op() == JSConstant) {
4886             JSValue value = m_graph.valueOfJSConstant(edge.node());
4887             if (!value.isCell()) {
4888                 terminate(Uncountable);
4889                 return m_out.intPtrZero;
4890             }
4891             return m_out.constIntPtr(value.asCell());
4892         }
4893
4894         LoweredNodeValue value = m_jsValueValues.get(edge.node());
4895         if (isValid(value)) {
4896             LValue uncheckedValue = value.value();
4897             FTL_TYPE_CHECK(
4898                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
4899             return uncheckedValue;
4900         }
4901
4902         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell));
4903         terminate(Uncountable);
4904         return m_out.intPtrZero;
4905     }
4906
4907     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4908     {
4909         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
4910
4911         LValue result = lowCell(edge, mode);
4912         speculateObject(edge, result);
4913         return result;
4914     }
4915
4916     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4917     {
4918         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse);
4919
4920         LValue result = lowCell(edge, mode);
4921         speculateString(edge, result);
4922         return result;
4923     }
4924
4925     LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4926     {
4927         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse);
4928
4929         LValue string = lowString(edge, mode);
4930         LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
4931         speculateStringIdent(edge, string, stringImpl);
4932         return stringImpl;
4933     }
4934
4935     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4936     {
4937         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
4938
4939         LValue result = lowCell(edge, mode);
4940         speculateNonNullObject(edge, result);
4941         return result;
4942     }
4943
4944     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4945     {
4946         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
4947
4948         if (edge->hasConstant()) {
4949             JSValue value = m_graph.valueOfJSConstant(edge.node());
4950             if (!value.isBoolean()) {
4951                 terminate(Uncountable);
4952                 return m_out.booleanFalse;
4953             }
4954             return m_out.constBool(value.asBoolean());
4955         }
4956
4957         LoweredNodeValue value = m_booleanValues.get(edge.node());
4958         if (isValid(value))
4959             return value.value();
4960
4961         value = m_jsValueValues.get(edge.node());
4962         if (isValid(value)) {
4963             LValue unboxedResult = value.value();
4964             FTL_TYPE_CHECK(
4965                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
4966             LValue result = unboxBoolean(unboxedResult);
4967             setBoolean(edge.node(), result);
4968             return result;
4969         }
4970
4971         RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean));
4972         terminate(Uncountable);
4973         return m_out.booleanFalse;
4974     }
4975
4976     LValue lowDouble(Edge edge)
4977     {
4978         RELEASE_ASSERT(isDouble(edge.useKind()));
4979
4980         LoweredNodeValue value = m_doubleValues.get(edge.node());
4981         if (isValid(value))
4982             return value.value();
4983
4984         RELEASE_ASSERT(!m_state.forNode(edge).m_type);
4985         terminate(Uncountable);
4986         return m_out.doubleZero;
4987     }
4988
4989     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
4990     {
4991         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
4992         RELEASE_ASSERT(!isDouble(edge.useKind()));
4993         RELEASE_ASSERT(edge.useKind() != Int52RepUse);
4994
4995         if (edge->hasConstant())
4996             return m_out.constInt64(JSValue::encode(m_graph.valueOfJSConstant(edge.node())));
4997
4998         LoweredNodeValue value = m_jsValueValues.get(edge.node());
4999         if (isValid(value))
5000             return value.value();
5001
5002         value = m_int32Values.get(edge.node());
5003         if (isValid(value)) {
5004             LValue result = boxInt32(value.value());
5005             setJSValue(edge.node(), result);
5006             return result;
5007         }
5008
5009         value = m_booleanValues.get(edge.node());
5010         if (isValid(value)) {
5011             LValue result = boxBoolean(value.value());
5012             setJSValue(edge.node(), result);
5013             return result;
5014         }
5015
5016         RELEASE_ASSERT_NOT_REACHED();
5017         return 0;
5018     }
5019
5020     LValue lowStorage(Edge edge)
5021     {
5022         LoweredNodeValue value = m_storageValues.get(edge.node());
5023         if (isValid(value))
5024             return value.value();
5025
5026         LValue result = lowCell(edge);
5027         setStorage(edge.node(), result);
5028         return result;
5029     }
5030
5031     LValue strictInt52ToInt32(Edge edge, LValue value)
5032     {
5033         LValue result = m_out.castToInt32(value);
5034         FTL_TYPE_CHECK(
5035             noValue(), edge, SpecInt32,
5036             m_out.notEqual(m_out.signExt(result, m_out.int64), value));
5037         setInt32(edge.node(), result);
5038         return result;
5039     }
5040
5041     LValue strictInt52ToDouble(LValue value)
5042     {
5043         return m_out.intToDouble(value);
5044     }
5045
5046     LValue strictInt52ToJSValue(LValue value)
5047     {
5048         LBasicBlock isInt32 = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isInt32 case"));
5049         LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isDouble case"));
5050         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue continuation"));
5051
5052         Vector<ValueFromBlock, 2> results;
5053
5054         LValue int32Value = m_out.castToInt32(value);
5055         m_out.branch(
5056             m_out.equal(m_out.signExt(int32Value, m_out.int64), value),
5057             unsure(isInt32), unsure(isDouble));
5058
5059         LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble);
5060
5061         results.append(m_out.anchor(boxInt32(int32Value)));
5062         m_out.jump(continuation);
5063
5064         m_out.appendTo(isDouble, continuation);
5065
5066         results.append(m_out.anchor(boxDouble(m_out.intToDouble(value))));
5067         m_out.jump(continuation);
5068
5069         m_out.appendTo(continuation, lastNext);
5070         return m_out.phi(m_out.int64, results);
5071     }
5072
5073     LValue strictInt52ToInt52(LValue value)
5074     {
5075         return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount));
5076     }
5077
5078     LValue int52ToStrictInt52(LValue value)
5079     {
5080         return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount));
5081     }
5082
5083     LValue isNotInt32(LValue jsValue)
5084     {
5085         return m_out.below(jsValue, m_tagTypeNumber);
5086     }
5087     LValue unboxInt32(LValue jsValue)
5088     {
5089         return m_out.castToInt32(jsValue);
5090     }
5091     LValue boxInt32(LValue value)
5092     {
5093         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
5094     }
5095
5096     LValue isCellOrMisc(LValue jsValue)
5097     {
5098         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
5099     }
5100     LValue isNotCellOrMisc(LValue jsValue)
5101     {
5102         return m_out.testNonZero64(jsValue, m_tagTypeNumber);
5103     }
5104
5105     LValue unboxDouble(LValue jsValue)
5106     {
5107         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
5108     }
5109     LValue boxDouble(LValue doubleValue)
5110     {
5111         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
5112     }
5113     LValue jsValueToDouble(Edge edge, LValue boxedValue)
5114     {
5115         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing int case"));
5116         LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing double case"));
5117         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing continuation"));
5118
5119         LValue isNotInt32;
5120         if (!m_interpreter.needsTypeCheck(edge, SpecInt32))
5121             isNotInt32 = m_out.booleanFalse;
5122         else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32))
5123             isNotInt32 = m_out.booleanTrue;
5124         else
5125             isNotInt32 = this->isNotInt32(boxedValue);
5126         m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
5127
5128         LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
5129
5130         ValueFromBlock intToDouble = m_out.anchor(
5131             m_out.intToDouble(unboxInt32(boxedValue)));
5132         m_out.jump(continuation);
5133
5134         m_out.appendTo(doubleCase, continuation);
5135
5136         FTL_TYPE_CHECK(
5137             jsValueValue(boxedValue), edge, SpecBytecodeNumber, isCellOrMisc(boxedValue));
5138
5139         ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedValue));
5140         m_out.jump(continuation);
5141
5142         m_out.appendTo(continuation, lastNext);
5143
5144         return m_out.phi(m_out.doubleType, intToDouble, unboxedDouble);
5145     }
5146
5147     LValue jsValueToStrictInt52(Edge edge, LValue boxedValue)
5148     {
5149         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing int case"));
5150         LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing double case"));
5151         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing continuation"));
5152
5153         LValue isNotInt32;
5154         if (!m_interpreter.needsTypeCheck(edge, SpecInt32))
5155             isNotInt32 = m_out.booleanFalse;
5156         else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32))
5157             isNotInt32 = m_out.booleanTrue;
5158         else
5159             isNotInt32 = this->isNotInt32(boxedValue);
5160         m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
5161
5162         LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
5163
5164         ValueFromBlock intToInt52 = m_out.anchor(
5165             m_out.signExt(unboxInt32(boxedValue), m_out.int64));
5166         m_out.jump(continuation);
5167
5168         m_out.appendTo(doubleCase, continuation);
5169
5170         LValue possibleResult = m_out.call(
5171             m_out.operation(operationConvertBoxedDoubleToInt52), boxedValue);
5172         FTL_TYPE_CHECK(
5173             jsValueValue(boxedValue), edge, SpecInt32 | SpecInt52AsDouble,
5174             m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
5175
5176         ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult);
5177         m_out.jump(continuation);
5178
5179         m_out.appendTo(continuation, lastNext);
5180
5181         return m_out.phi(m_out.int64, intToInt52, doubleToInt52);
5182     }
5183
5184     LValue doubleToStrictInt52(Edge edge, LValue value)
5185     {
5186         LValue possibleResult = m_out.call(
5187             m_out.operation(operationConvertDoubleToInt52), value);
5188         FTL_TYPE_CHECK(
5189             doubleValue(value), edge, SpecInt52AsDouble,
5190             m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
5191
5192         return possibleResult;
5193     }
5194
5195     LValue isNumber(LValue jsValue)
5196     {
5197         return isNotCellOrMisc(jsValue);
5198     }
5199     LValue isNotNumber(LValue jsValue)
5200     {
5201         return isCellOrMisc(jsValue);
5202     }
5203
5204     LValue isNotCell(LValue jsValue)
5205     {
5206         return m_out.testNonZero64(jsValue, m_tagMask);
5207     }
5208
5209     LValue isCell(LValue jsValue)
5210     {
5211         return m_out.testIsZero64(jsValue, m_tagMask);
5212     }
5213
5214     LValue isNotMisc(LValue value)
5215     {
5216         return m_out.above(value, m_out.constInt64(TagBitTypeOther | TagBitBool | TagBitUndefined));
5217     }
5218
5219     LValue isMisc(LValue value)
5220     {
5221         return m_out.bitNot(isNotMisc(value));
5222     }
5223
5224     LValue isNotBoolean(LValue jsValue)
5225     {
5226         return m_out.testNonZero64(
5227             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
5228             m_out.constInt64(~1));
5229     }
5230     LValue isBoolean(LValue jsValue)
5231     {
5232         return m_out.bitNot(isNotBoolean(jsValue));
5233     }
5234     LValue unboxBoolean(LValue jsValue)
5235     {
5236         // We want to use a cast that guarantees that LLVM knows that even the integer
5237         // value is just 0 or 1. But for now we do it the dumb way.
5238         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
5239     }
5240     LValue boxBoolean(LValue value)
5241     {
5242         return m_out.select(
5243             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
5244     }
5245
5246     LValue isNotOther(LValue value)
5247     {
5248         return m_out.notEqual(
5249             m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
5250             m_out.constInt64(ValueNull));
5251     }
5252     LValue isOther(LValue value)
5253     {
5254         return m_out.equal(
5255             m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
5256             m_out.constInt64(ValueNull));
5257     }
5258
5259     void speculate(Edge edge)
5260     {
5261         switch (edge.useKind()) {
5262         case UntypedUse:
5263             break;
5264         case KnownInt32Use:
5265         case KnownStringUse:
5266         case DoubleRepUse:
5267         case Int52RepUse:
5268             ASSERT(!m_interpreter.needsTypeCheck(edge));
5269             break;
5270         case Int32Use:
5271             speculateInt32(edge);
5272             break;
5273         case CellUse:
5274             speculateCell(edge);
5275             break;
5276         case KnownCellUse:
5277             ASSERT(!m_interpreter.needsTypeCheck(edge));
5278             break;
5279         case MachineIntUse:
5280             speculateMachineInt(edge);
5281             break;
5282         case ObjectUse:
5283             speculateObject(edge);
5284             break;
5285         case ObjectOrOtherUse:
5286             speculateObjectOrOther(edge);
5287             break;
5288         case FinalObjectUse:
5289             speculateFinalObject(edge);
5290             break;
5291         case StringUse:
5292             speculateString(edge);
5293             break;
5294         case StringIdentUse:
5295             speculateStringIdent(edge);
5296             break;
5297         case StringObjectUse:
5298             speculateStringObject(edge);
5299             break;
5300         case StringOrStringObjectUse:
5301             speculateStringOrStringObject(edge);
5302             break;
5303         case NumberUse:
5304             speculateNumber(edge);
5305             break;
5306         case DoubleRepRealUse:
5307             speculateDoubleReal(edge);
5308             break;
5309         case DoubleRepMachineIntUse:
5310             speculateDoubleRepMachineInt(edge);
5311             break;
5312         case BooleanUse:
5313             speculateBoolean(edge);
5314             break;
5315         case NotStringVarUse:
5316             speculateNotStringVar(edge);
5317             break;
5318         case NotCellUse:
5319             speculateNotCell(edge);
5320             break;
5321         case OtherUse:
5322             speculateOther(edge);
5323             break;
5324         case MiscUse:
5325             speculateMisc(edge);
5326             break;
5327         default:
5328             dataLog("Unsupported speculation use kind: ", edge.useKind(), "\n");
5329             RELEASE_ASSERT_NOT_REACHED();
5330         }
5331     }
5332
5333     void speculate(Node*, Edge edge)
5334     {
5335         speculate(edge);
5336     }
5337
5338     void speculateInt32(Edge edge)
5339     {
5340         lowInt32(edge);
5341     }
5342
5343     void speculateCell(Edge edge)
5344     {
5345         lowCell(edge);
5346     }
5347
5348     void speculateMachineInt(Edge edge)
5349     {
5350         if (!m_interpreter.needsTypeCheck(edge))
5351             return;
5352
5353         jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation));
5354     }
5355
5356     LValue isObject(LValue cell)
5357     {
5358         return m_out.notEqual(
5359             m_out.load32(cell, m_heaps.JSCell_structureID),
5360             m_out.constInt32(vm().stringStructure->id()));
5361     }
5362
5363     LValue isNotString(LValue cell)
5364     {
5365         return isObject(cell);
5366     }
5367
5368     LValue isString(LValue cell)
5369     {
5370         return m_out.equal(
5371             m_out.load32(cell, m_heaps.JSCell_structureID),
5372             m_out.constInt32(vm().stringStructure->id()));
5373     }
5374
5375     LValue isNotObject(LValue cell)
5376     {
5377         return isString(cell);
5378     }
5379
5380     LValue isArrayType(LValue cell, ArrayMode arrayMode)
5381     {
5382         switch (arrayMode.type()) {
5383         case Array::Int32:
5384         case Array::Double:
5385         case Array::Contiguous: {
5386             LValue indexingType = m_out.load8(cell, m_heaps.JSCell_indexingType);
5387
5388             switch (arrayMode.arrayClass()) {
5389             case Array::OriginalArray:
5390                 RELEASE_ASSERT_NOT_REACHED();
5391                 return 0;
5392
5393             case Array::Array:
5394                 return m_out.equal(
5395                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
5396                     m_out.constInt8(IsArray | arrayMode.shapeMask()));
5397
5398             case Array::NonArray:
5399             case Array::OriginalNonArray:
5400                 return m_out.equal(
5401                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
5402                     m_out.constInt8(arrayMode.shapeMask()));
5403
5404             case Array::PossiblyArray:
5405                 return m_out.equal(
5406                     m_out.bitAnd(indexingType, m_out.constInt8(IndexingShapeMask)),
5407                     m_out.constInt8(arrayMode.shapeMask()));
5408             }
5409
5410             RELEASE_ASSERT_NOT_REACHED();
5411         }
5412
5413         default:
5414             return m_out.equal(
5415                 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
5416                 m_out.constInt8(typeForTypedArrayType(arrayMode.typedArrayType())));
5417         }
5418     }
5419
5420     LValue isType(LValue cell, JSType type)
5421     {
5422         return m_out.equal(
5423             m_out.load8(cell, m_heaps.JSCell_typeInfoType),
5424             m_out.constInt8(type));
5425     }
5426
5427     LValue isNotType(LValue cell, JSType type)
5428     {
5429         return m_out.bitNot(isType(cell, type));
5430     }
5431
5432     void speculateObject(Edge edge, LValue cell)
5433     {
5434         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
5435     }
5436
5437     void speculateObject(Edge edge)
5438     {
5439         speculateObject(edge, lowCell(edge));
5440     }
5441
5442     void speculateObjectOrOther(Edge edge)
5443     {
5444         if (!m_interpreter.needsTypeCheck(edge))
5445             return;
5446
5447         LValue value = lowJSValue(edge);
5448
5449         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case"));
5450         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case"));
5451         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation"));
5452
5453         m_out.branch(isNotCell(value), unsure(primitiveCase), unsure(cellCase));
5454
5455         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
5456
5457         FTL_TYPE_CHECK(
5458             jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value));
5459
5460         m_out.jump(continuation);
5461
5462         m_out.appendTo(primitiveCase, continuation);
5463
5464         FTL_TYPE_CHECK(
5465             jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
5466
5467         m_out.jump(continuation);
5468
5469         m_out.appendTo(continuation, lastNext);
5470     }
5471
5472     void speculateFinalObject(Edge edge, LValue cell)
5473     {
5474         FTL_TYPE_CHECK(
5475             jsValueValue(cell), edge, SpecFinalObject, isNotType(cell, FinalObjectType));
5476     }
5477
5478     void speculateFinalObject(Edge edge)
5479     {
5480         speculateFinalObject(edge, lowCell(edge));
5481     }
5482
5483     void speculateString(Edge edge, LValue cell)
5484     {
5485         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString | ~SpecCell, isNotString(cell));
5486     }
5487
5488     void speculateString(Edge edge)
5489     {
5490         speculateString(edge, lowCell(edge));
5491     }
5492
5493     void speculateStringIdent(Edge edge, LValue string, LValue stringImpl)
5494     {
5495         if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString))
5496             return;
5497
5498         speculate(BadType, jsValueValue(string), edge.node(), m_out.isNull(stringImpl));
5499         speculate(
5500             BadType, jsValueValue(string), edge.node(),
5501             m_out.testIsZero32(
5502                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
5503                 m_out.constInt32(StringImpl::flagIsAtomic())));
5504         m_interpreter.filter(edge, SpecStringIdent | ~SpecString);
5505     }
5506
5507     void speculateStringIdent(Edge edge)
5508     {
5509         lowStringIdent(edge);
5510     }
5511
5512     void speculateStringObject(Edge edge)
5513     {
5514         if (!m_interpreter.needsTypeCheck(edge, SpecStringObject))
5515             return;
5516
5517         speculateStringObjectForCell(edge, lowCell(edge));
5518         m_interpreter.filter(edge, SpecStringObject);
5519     }
5520
5521     void speculateStringOrStringObject(Edge edge)
5522     {
5523         if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject))
5524             return;
5525
5526         LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject not string case"));
5527         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject continuation"));
5528
5529         LValue structureID = m_out.load32(lowCell(edge), m_heaps.JSCell_structureID);
5530         m_out.branch(
5531             m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
5532             unsure(continuation), unsure(notString));
5533
5534         LBasicBlock lastNext = m_out.appendTo(notString, continuation);
5535         speculateStringObjectForStructureID(edge, structureID);
5536         m_out.jump(continuation);
5537
5538         m_out.appendTo(continuation, lastNext);
5539
5540         m_interpreter.filter(edge, SpecString | SpecStringObject);
5541     }
5542
5543     void speculateStringObjectForCell(Edge edge, LValue cell)
5544     {
5545         speculateStringObjectForStructureID(edge, m_out.load32(cell, m_heaps.JSCell_structureID));
5546     }
5547
5548     void speculateStringObjectForStructureID(Edge edge, LValue structureID)
5549     {
5550         Structure* stringObjectStructure =
5551             m_graph.globalObjectFor(m_node->origin.semantic)->stringObjectStructure();
5552
5553         if (m_state.forNode(edge).m_currentKnownStructure.isSubsetOf(StructureSet(stringObjectStructure)))
5554             return;
5555
5556         speculate(
5557             NotStringObject, noValue(), 0,
5558             m_out.notEqual(structureID, weakStructure(stringObjectStructure)));
5559     }
5560
5561     void speculateNonNullObject(Edge edge, LValue cell)
5562     {
5563         FTL_TYPE_CHECK(
5564             jsValueValue(cell), edge, SpecObject,
5565             m_out.equal(
5566                 m_out.load32(cell, m_heaps.JSCell_structureID),
5567                 m_out.constInt32(vm().stringStructure->id())));
5568         if (masqueradesAsUndefinedWatchpointIsStillValid())
5569             return;
5570
5571         speculate(
5572             BadType, jsValueValue(cell), edge.node(),
5573             m_out.testNonZero8(
5574                 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
5575                 m_out.constInt8(MasqueradesAsUndefined)));
5576     }
5577
5578     void speculateNumber(Edge edge)
5579     {
5580         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5581         FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value));
5582     }
5583
5584     void speculateDoubleReal(Edge edge)
5585     {
5586         // Do an early return here because lowDouble() can create a lot of control flow.
5587         if (!m_interpreter.needsTypeCheck(edge))
5588             return;
5589
5590         LValue value = lowDouble(edge);
5591         FTL_TYPE_CHECK(
5592             doubleValue(value), edge, SpecDoubleReal,
5593             m_out.doubleNotEqualOrUnordered(value, value));
5594     }
5595
5596     void speculateDoubleRepMachineInt(Edge edge)
5597     {
5598         if (!m_interpreter.needsTypeCheck(edge))
5599             return;
5600
5601         doubleToStrictInt52(edge, lowDouble(edge));
5602     }
5603
5604     void speculateBoolean(Edge edge)
5605     {
5606         lowBoolean(edge);
5607     }
5608
5609     void speculateNotStringVar(Edge edge)
5610     {
5611         if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar))
5612             return;
5613
5614         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5615
5616         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is cell case"));
5617         LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is string case"));
5618         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar continuation"));
5619
5620         m_out.branch(isCell(value), unsure(isCellCase), unsure(continuation));
5621
5622         LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
5623         m_out.branch(isString(value), unsure(isStringCase), unsure(continuation));
5624
5625         m_out.appendTo(isStringCase, continuation);
5626         speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value));
5627         m_out.jump(continuation);
5628
5629         m_out.appendTo(continuation, lastNext);
5630     }
5631
5632     void speculateNotCell(Edge edge)
5633     {
5634         if (!m_interpreter.needsTypeCheck(edge))
5635             return;
5636
5637         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5638         typeCheck(jsValueValue(value), edge, ~SpecCell, isCell(value));
5639     }
5640
5641     void speculateOther(Edge edge)
5642     {
5643         if (!m_interpreter.needsTypeCheck(edge))
5644             return;
5645
5646         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5647         typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value));
5648     }
5649
5650     void speculateMisc(Edge edge)
5651     {
5652         if (!m_interpreter.needsTypeCheck(edge))
5653             return;
5654
5655         LValue value = lowJSValue(edge, ManualOperandSpeculation);
5656         typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value));
5657     }
5658
5659     bool masqueradesAsUndefinedWatchpointIsStillValid()
5660     {
5661         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic);
5662     }
5663
5664     LValue loadMarkByte(LValue base)
5665     {
5666         return m_out.load8(base, m_heaps.JSCell_gcData);
5667     }
5668
5669     void emitStoreBarrier(LValue base)
5670     {
5671 #if ENABLE(GGC)
5672         LBasicBlock isMarkedAndNotRemembered = FTL_NEW_BLOCK(m_out, ("Store barrier is marked block"));
5673         LBasicBlock bufferHasSpace = FTL_NEW_BLOCK(m_out, ("Store barrier buffer has space"));
5674         LBasicBlock bufferIsFull = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full"));
5675         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation"));
5676
5677         // Check the mark byte.
5678         m_out.branch(
5679             m_out.notZero8(loadMarkByte(base)), usually(continuation), rarely(isMarkedAndNotRemembered));
5680
5681         // Append to the write barrier buffer.
5682         LBasicBlock lastNext = m_out.appendTo(isMarkedAndNotRemembered, bufferHasSpace);
5683         LValue currentBufferIndex = m_out.load32(m_out.absolute(&vm().heap.writeBarrierBuffer().m_currentIndex));
5684         LValue bufferCapacity = m_out.load32(m_out.absolute(&vm().heap.writeBarrierBuffer().m_capacity));
5685         m_out.branch(
5686             m_out.lessThan(currentBufferIndex, bufferCapacity),
5687             usually(bufferHasSpace), rarely(bufferIsFull));
5688
5689         // Buffer has space, store to it.
5690         m_out.appendTo(bufferHasSpace, bufferIsFull);
5691         LValue writeBarrierBufferBase = m_out.loadPtr(m_out.absolute(&vm().heap.writeBarrierBuffer().m_buffer));
5692         m_out.storePtr(base, m_out.baseIndex(m_heaps.WriteBarrierBuffer_bufferContents, writeBarrierBufferBase, m_out.zeroExt(currentBufferIndex, m_out.intPtr), ScalePtr));
5693         m_out.store32(m_out.add(currentBufferIndex, m_out.constInt32(1)), m_out.absolute(&vm().heap.writeBarrierBuffer().m_currentIndex));
5694         m_out.jump(continuation);
5695
5696         // Buffer is out of space, flush it.
5697         m_out.appendTo(bufferIsFull, continuation);
5698         vmCall(m_out.operation(operationFlushWriteBarrierBuffer), m_callFrame, base, NoExceptions);
5699         m_out.jump(continuation);
5700
5701         m_out.appendTo(continuation, lastNext);
5702 #else
5703         UNUSED_PARAM(base);
5704 #endif
5705     }
5706
5707     enum ExceptionCheckMode { NoExceptions, CheckExceptions };
5708
5709     LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions)
5710     {
5711         callPreflight();
5712         LValue result = m_out.call(function);
5713         callCheck(mode);
5714         return result;
5715     }
5716     LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions)
5717     {
5718         callPreflight();
5719         LValue result = m_out.call(function, arg1);
5720         callCheck(mode);
5721         return result;
5722     }
5723     LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions)
5724     {
5725         callPreflight();
5726         LValue result = m_out.call(function, arg1, arg2);
5727         callCheck(mode);
5728         return result;
5729     }
5730     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions)
5731     {
5732         callPreflight();
5733         LValue result = m_out.call(function, arg1, arg2, arg3);
5734         callCheck(mode);
5735         return result;
5736     }
5737     LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions)
5738     {
5739         callPreflight();
5740         LValue result = m_out.call(function, arg1, arg2, arg3, arg4);
5741         callCheck(mode);
5742         return result;
5743     }
5744
5745     void callPreflight(CodeOrigin codeOrigin)
5746     {
5747         m_out.store32(
5748             m_out.constInt32(
5749                 CallFrame::Location::encodeAsCodeOriginIndex(
5750                     m_ftlState.jitCode->common.addCodeOrigin(codeOrigin))),
5751             tagFor(JSStack::ArgumentCount));
5752     }
5753     void callPreflight()
5754     {
5755         callPreflight(m_node->origin.semantic);
5756     }
5757
5758     void callCheck(ExceptionCheckMode mode = CheckExceptions)
5759     {
5760         if (mode == NoExceptions)
5761             return;
5762
5763         if (Options::enableExceptionFuzz())
5764             m_out.call(m_out.operation(operationExceptionFuzz));
5765
5766         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
5767
5768         m_out.branch(
5769             m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))),
5770             rarely(m_handleExceptions), usually(continuation));
5771
5772         m_out.appendTo(continuation);
5773     }
5774
5775     LBasicBlock lowBlock(BasicBlock* block)
5776     {
5777         return m_blocks.get(block);
5778     }
5779
5780     void initializeOSRExitStateForBlock()
5781     {
5782         m_availability = m_highBlock->ssa->availabilityAtHead;
5783     }
5784
5785     void appendOSRExit(
5786         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
5787     {
5788         if (verboseCompilationEnabled()) {
5789             dataLog("    OSR exit #", m_ftlState.jitCode->osrExit.size(), " with availability: ", m_availability, "\n");
5790             if (!m_availableRecoveries.isEmpty())
5791                 dataLog("        Available recoveries: ", listDump(m_availableRecoveries), "\n");
5792         }
5793
5794         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.finalizer->osrExit.size());
5795
5796         m_ftlState.jitCode->osrExit.append(OSRExit(
5797             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
5798             m_codeOriginForExitTarget, m_codeOriginForExitProfile,
5799             m_availability.numberOfArguments(), m_availability.numberOfLocals()));
5800         m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
5801
5802         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
5803
5804         LBasicBlock lastNext = 0;
5805         LBasicBlock continuation = 0;
5806
5807         LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
5808         continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
5809
5810         m_out.branch(failCondition, rarely(failCase), usually(continuation));
5811
5812         lastNext = m_out.appendTo(failCase, continuation);
5813
5814         emitOSRExitCall(exit, lowValue);
5815
5816         m_out.unreachable();
5817
5818         m_out.appendTo(continuation, lastNext);
5819     }
5820
5821     void emitOSRExitCall(OSRExit& exit, FormattedValue lowValue)
5822     {
5823         ExitArgumentList arguments;
5824
5825         CodeOrigin codeOrigin = exit.m_codeOrigin;
5826
5827         buildExitArguments(exit, arguments, lowValue, codeOrigin);
5828
5829         callStackmap(exit, arguments);
5830     }
5831
5832     void buildExitArguments(
5833         OSRExit& exit, ExitArgumentList& arguments, FormattedValue lowValue,
5834         CodeOrigin codeOrigin)
5835     {
5836         if (!!lowValue)
5837             arguments.append(lowValue.value());
5838
5839         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
5840             int operand = exit.m_values.operandForIndex(i);
5841             bool isLive = m_graph.isLiveInBytecode(VirtualRegister(operand), codeOrigin);
5842             if (!isLive) {
5843                 exit.m_values[i] = ExitValue::dead();
5844                 continue;
5845             }
5846
5847             Availability availability = m_availability[i];
5848             FlushedAt flush = availability.flushedAt();
5849             switch (flush.format()) {
5850             case DeadFlush:
5851             case ConflictingFlush:
5852                 if (availability.hasNode()) {
5853                     addExitArgumentForNode(exit, arguments, i, availability.node());
5854                     break;
5855                 }
5856
5857                 if (Options::validateFTLOSRExitLiveness()) {
5858                     dataLog("Expected r", operand, " to be available but it wasn't.\n");
5859                     RELEASE_ASSERT_NOT_REACHED();
5860                 }
5861
5862                 // This means that the DFG's DCE proved that the value is dead in bytecode
5863                 // even though the bytecode liveness analysis thinks it's live. This is
5864                 // acceptable since the DFG's DCE is by design more aggressive while still
5865                 // being sound.
5866                 exit.m_values[i] = ExitValue::dead();
5867                 break;
5868
5869             case FlushedJSValue:
5870             case FlushedCell:
5871             case FlushedBoolean:
5872                 exit.m_values[i] = ExitValue::inJSStack(flush.virtualRegister());
5873                 break;
5874
5875             case FlushedInt32:
5876                 exit.m_values[i] = ExitValue::inJSStackAsInt32(flush.virtualRegister());
5877                 break;
5878
5879             case FlushedInt52:
5880                 exit.m_values[i] = ExitValue::inJSStackAsInt52(flush.virtualRegister());
5881                 break;
5882
5883             case FlushedDouble:
5884                 exit.m_values[i] = ExitValue::inJSStackAsDouble(flush.virtualRegister());
5885                 break;
5886
5887             case FlushedArguments:
5888                 exit.m_values[i] = ExitValue::argumentsObjectThatWasNotCreated();
5889                 break;
5890             }
5891         }
5892
5893         if (verboseCompilationEnabled())
5894             dataLog("        Exit values: ", exit.m_values, "\n");
5895     }
5896
5897     void callStackmap(OSRExit& exit, ExitArgumentList& arguments)
5898     {
5899         exit.m_stackmapID = m_stackmapIDs++;
5900         arguments.insert(0, m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
5901         arguments.insert(0, m_out.constInt64(exit.m_stackmapID));
5902
5903         m_out.call(m_out.stackmapIntrinsic(), arguments);
5904     }
5905
5906     void addExitArgumentForNode(
5907         OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node)
5908     {
5909         ASSERT(node->shouldGenerate());
5910         ASSERT(node->hasResult());
5911
5912         if (tryToSetConstantExitArgument(exit, index, node))
5913             return;
5914
5915         for (unsigned i = 0; i < m_availableRecoveries.size(); ++i) {
5916             AvailableRecovery recovery = m_availableRecoveries[i];
5917             if (recovery.node() != node)
5918                 continue;
5919
5920             exit.m_values[index] = ExitValue::recovery(
5921                 recovery.opcode(), arguments.size(), arguments.size() + 1,
5922                 recovery.format());
5923             arguments.append(recovery.left());
5924             arguments.append(recovery.right());
5925             return;
5926         }
5927
5928         LoweredNodeValue value = m_int32Values.get(node);
5929         if (isValid(value)) {
5930             addExitArgument(exit, arguments, index, ValueFormatInt32, value.value());
5931             return;
5932         }
5933
5934         value = m_int52Values.get(node);
5935         if (isValid(value)) {
5936             addExitArgument(exit, arguments, index, ValueFormatInt52, value.value());
5937             return;
5938         }
5939
5940         value = m_strictInt52Values.get(node);
5941         if (isValid(value)) {
5942             addExitArgument(exit, arguments, index, ValueFormatStrictInt52, value.value());
5943             return;
5944         }
5945
5946         value = m_booleanValues.get(node);
5947         if (isValid(value)) {
5948             LValue valueToPass = m_out.zeroExt(value.value(), m_out.int32);
5949             addExitArgument(exit, arguments, index, ValueFormatBoolean, valueToPass);
5950             return;
5951         }
5952
5953         value = m_jsValueValues.get(node);
5954         if (isValid(value)) {
5955             addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value());
5956             return;
5957         }
5958
5959         value = m_doubleValues.get(node);
5960         if (isValid(value)) {
5961             addExitArgument(exit, arguments, index, ValueFormatDouble, value.value());
5962             return;
5963         }
5964
5965         startCrashing();
5966         dataLog("Cannot find value for node: ", node, " while compiling exit at ", exit.m_codeOrigin, " in node ", m_node, "\n");
5967         m_graph.dump();
5968         RELEASE_ASSERT_NOT_REACHED();
5969     }
5970
5971     bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node)
5972     {
5973         if (!node)
5974             return false;
5975
5976         switch (node->op()) {
5977         case JSConstant:
5978         case Int52Constant:
5979         case DoubleConstant:
5980         case WeakJSConstant:
5981             exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node));
5982             return true;
5983         case PhantomArguments:
5984             exit.m_values[index] = ExitValue::argumentsObjectThatWasNotCreated();
5985             return true;
5986         default:
5987             return false;
5988         }
5989     }
5990
5991     void addExitArgument(
5992         OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format,
5993         LValue value)
5994     {
5995         exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
5996         arguments.append(value);
5997     }
5998
5999     bool doesKill(Edge edge)
6000     {
6001         if (edge.doesNotKill())
6002             return false;
6003
6004         if (edge->hasConstant())
6005             return false;
6006
6007         return true;
6008     }
6009
6010     void addAvailableRecovery(
6011         Node* node, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
6012     {
6013         m_availableRecoveries.append(AvailableRecovery(node, opcode, left, right, format));
6014     }
6015
6016     void addAvailableRecovery(
6017         Edge edge, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
6018     {
6019         addAvailableRecovery(edge.node(), opcode, left, right, format);
6020     }
6021
6022     void setInt32(Node* node, LValue value)
6023     {
6024         m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
6025     }
6026     void setInt52(Node* node, LValue value)
6027     {
6028         m_int52Values.set(node, LoweredNodeValue(value, m_highBlock));
6029     }
6030     void setStrictInt52(Node* node, LValue value)
6031     {
6032         m_strictInt52Values.set(node, LoweredNodeValue(value, m_highBlock));
6033     }
6034     void setInt52(Node* node, LValue value, Int52Kind kind)
6035     {
6036         switch (kind) {
6037         case Int52:
6038             setInt52(node, value);
6039             return;
6040
6041         case StrictInt52:
6042             setStrictInt52(node, value);
6043             return;
6044         }
6045
6046         RELEASE_ASSERT_NOT_REACHED();
6047     }
6048     void setJSValue(Node* node, LValue value)
6049     {
6050         m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
6051     }
6052     void setBoolean(Node* node, LValue value)
6053     {
6054         m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
6055     }
6056     void setStorage(Node* node, LValue value)
6057     {
6058         m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
6059     }
6060     void setDouble(Node* node, LValue value)
6061     {
6062         m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
6063     }
6064
6065     void setInt32(LValue value)
6066     {
6067         setInt32(m_node, value);
6068     }
6069     void setInt52(LValue value)
6070     {
6071         setInt52(m_node, value);
6072     }
6073     void setStrictInt52(LValue value)
6074     {
6075         setStrictInt52(m_node, value);
6076     }
6077     void setInt52(LValue value, Int52Kind kind)
6078     {
6079         setInt52(m_node, value, kind);
6080     }
6081     void setJSValue(LValue value)
6082     {
6083         setJSValue(m_node, value);
6084     }
6085     void setBoolean(LValue value)
6086     {
6087         setBoolean(m_node, value);
6088     }
6089     void setStorage(LValue value)
6090     {
6091         setStorage(m_node, value);
6092     }
6093     void setDouble(LValue value)
6094     {
6095         setDouble(m_node, value);
6096     }
6097
6098     bool isValid(const LoweredNodeValue& value)
6099     {
6100         if (!value)
6101             return false;
6102         if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
6103             return false;
6104         return true;
6105     }
6106
6107     void addWeakReference(JSCell* target)
6108     {
6109         m_graph.m_plan.weakReferences.addLazily(target);
6110     }
6111
6112     LValue loadStructure(LValue value)
6113     {
6114         LValue tableIndex = m_out.load32(value, m_heaps.JSCell_structureID);
6115         LValue tableBase = m_out.loadPtr(
6116             m_out.absolute(vm().heap.structureIDTable().base()));
6117         LValue pointerIntoTable = m_out.baseIndex(
6118             tableBase, m_out.zeroExt(tableIndex, m_out.intPtr), ScaleEight);
6119         return m_out.loadPtr(TypedPointer(m_heaps.structureTable, pointerIntoTable));
6120     }
6121
6122     LValue weakPointer(JSCell* pointer)
6123     {
6124         addWeakReference(pointer);
6125         return m_out.constIntPtr(pointer);
6126     }
6127
6128     LValue weakStructure(Structure* structure)
6129     {
6130         addWeakReference(structure);
6131         return m_out.constInt32(structure->id());
6132     }
6133
6134     TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
6135     {
6136         return m_out.address(base, m_heaps.variables[operand], offset);
6137     }
6138     TypedPointer payloadFor(LValue base, int operand)
6139     {
6140         return addressFor(base, operand, PayloadOffset);
6141     }
6142     TypedPointer tagFor(LValue base, int operand)
6143     {
6144         return addressFor(base, operand, TagOffset);
6145     }
6146     TypedPointer addressFor(int operand, ptrdiff_t offset = 0)
6147     {
6148         return addressFor(VirtualRegister(operand), offset);
6149     }
6150     TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0)
6151     {
6152         if (operand.isLocal())
6153             return addressFor(m_captured, operand.offset(), offset);
6154         return addressFor(m_callFrame, operand.offset(), offset);
6155     }
6156     TypedPointer payloadFor(int operand)
6157     {
6158         return payloadFor(VirtualRegister(operand));
6159     }
6160     TypedPointer payloadFor(VirtualRegister operand)
6161     {
6162         return addressFor(operand, PayloadOffset);
6163     }
6164     TypedPointer tagFor(int operand)
6165     {
6166         return tagFor(VirtualRegister(operand));
6167     }
6168     TypedPointer tagFor(VirtualRegister operand)
6169     {
6170         return addressFor(operand, TagOffset);
6171     }
6172
6173     VM& vm() { return m_graph.m_vm; }
6174     CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
6175
6176     Graph& m_graph;
6177     State& m_ftlState;
6178     AbstractHeapRepository m_heaps;
6179     Output m_out;
6180
6181     LBasicBlock m_prologue;
6182     LBasicBlock m_handleExceptions;
6183     HashMap<BasicBlock*, LBasicBlock> m_blocks;
6184
6185     LValue m_callFrame;
6186     LValue m_captured;
6187     LValue m_tagTypeNumber;
6188     LValue m_tagMask;
6189
6190     HashMap<Node*, LoweredNodeValue> m_int32Values;
6191     HashMap<Node*, LoweredNodeValue> m_strictInt52Values;
6192     HashMap<Node*, LoweredNodeValue> m_int52Values;
6193     HashMap<Node*, LoweredNodeValue> m_jsValueValues;
6194     HashMap<Node*, LoweredNodeValue> m_booleanValues;
6195     HashMap<Node*, LoweredNodeValue> m_storageValues;
6196     HashMap<Node*, LoweredNodeValue> m_doubleValues;
6197
6198     HashMap<Node*, LValue> m_phis;
6199
6200     Operands<Availability> m_availability;
6201
6202     Vector<AvailableRecovery, 3> m_availableRecoveries;
6203
6204     InPlaceAbstractState m_state;
6205     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
6206     BasicBlock* m_highBlock;
6207     BasicBlock* m_nextHighBlock;
6208     LBasicBlock m_nextLowBlock;
6209
6210     CodeOrigin m_codeOriginForExitTarget;
6211     CodeOrigin m_codeOriginForExitProfile;
6212     unsigned m_nodeIndex;
6213     Node* m_node;
6214
6215     uint32_t m_stackmapIDs;
6216 };
6217
6218 void lowerDFGToLLVM(State& state)
6219 {
6220     LowerDFGToLLVM lowering(state);
6221     lowering.lower();
6222 }
6223
6224 } } // namespace JSC::FTL
6225
6226 #endif // ENABLE(FTL_JIT)
6227